// Add a column with absolute mag import { type Result, ok, err, unwrap, unwrapOr, } from "./result"; import { base64ToBytes, binaryColumnWidth, binaryTypeSize, buildColumnIndex, bytesToAscii, classifyVotableType, computeMinMax, detectDelimiter, findErrCol, type FitsHeader, fitsDataLength, fitsPaddedLength, getXmlAttr, inferType, isDashRow, looksLikeVotableXml, parseBinaryTForm, parseColumnComments, parseVotableArraySize, parseVotableFields, parseVotableRow, readFitsHeader, splitCSVLine, type VotableField, votableElementSize, } from "./utils"; /// DATA /////////////////////////////////////////////////////////////////////////////////////////// /** parse result extended with a column index and per-column metadata, * generated from the file. */ export type Data = { mainData: string[][], columnIndex: ColumnIndex, columnMeta: Map, }; export const fileTypes = [".csv", ".tsv", ".fit", ".vot", ".b64", ".b264"] as const; export type FileType = typeof fileTypes[number]; export function parseRaw(raw: string | ArrayBuffer, type: FileType): Result { // Every parse function and every function they call whatever they throw is caught here // and turned into a real `Failure`, so every caller of `parseRaw` gets // an honest `Result` and never has to wrap it in try/catch itself. try { switch (type) { case ".csv": if (typeof raw !== "string") throw new Error(".csv files must be read as text (file.text()), not binary."); return ok(parseCSV(raw.split("\n"))); case ".tsv": if (typeof raw !== "string") throw new Error(".tsv files must be read as text (file.text()), not binary."); // Vizier export two type of .tsv, *-seperated, and csv + xml return ok(looksLikeVotableXml(raw) ? parseVOT(raw) : parseTSV(raw.split("\n"))); case ".fit": if (!(raw instanceof ArrayBuffer)) throw new Error(".fit files are binary (FITS) and must be read with file.arrayBuffer(), not file.text()."); return ok(parseFIT(raw)); case ".vot": case ".b64": case ".b264": if (typeof raw !== "string") throw new Error(".vot/.b64/.b264 files must be read as text (file.text()), not binary."); return ok(parseVOT(raw)); } } catch (caught) { return err(caught instanceof Error ? caught : new Error(String(caught))); } } //////////////////////////////////////////////////////////////////////////////////////////////////// /// PARSE ////////////////////////////////////////////////////////////////////////////////////////// /** * Parses a comma-separated file, * Assumes first non-blank row = headers, every row after that = data. */ function parseCSV(rawRows: string[]): Data { const lines = rawRows.filter(r => r.trim().length > 0); if (lines.length === 0) { throw new Error("The file is empty."); } const headers = splitCSVLine(lines[0]); if (headers.length === 0 || headers.every(h => h.trim() === "")) { throw new Error("No column headers were found in the first row."); } const mainData = lines.slice(1).map(splitCSVLine); if (mainData.length === 0) { throw new Error("The file only has a header row — no data rows were found."); } const columnIndex = buildColumnIndex(headers); const columnMeta = new Map(); headers.forEach((h, i) => { const values = mainData.map(row => row[i] ?? ""); const type = inferType(values); const { min, max } = computeMinMax(values, type); columnMeta.set(h, { type, description: null, unit: null, errorCol: findErrCol(headers, h), max, min, }); }); return { mainData, columnIndex, columnMeta }; } /** * Parses VizieR tsv files. * Structure: * - a block of `#` with metadata lines. The `#Column` lines describe * every output column. * - a blank line * - header row * - units row * - a row of dashes * - the data rows. * * VizieR lets users pick a preferred field separator * so the delimiter is auto-detected from the header row. */ function parseTSV(rawRows: string[]): Data { const columnDocs = parseColumnComments(rawRows); const tableLines = rawRows.filter(r => !r.startsWith("#")); let i = 0; while (i < tableLines.length && tableLines[i].trim() === "") i++; if (i >= tableLines.length) { throw new Error("No data table found in the file."); } const delimiter = detectDelimiter(tableLines[i]); const headers = tableLines[i].split(delimiter).map(h => h.trim()); if (headers.length === 0 || headers.every(h => h === "")) { throw new Error("No column headers were found in the file."); } i++; const unitsLine = i < tableLines.length ? tableLines[i] : ""; const units = unitsLine.split(delimiter).map(u => u.trim()); i++; if (i < tableLines.length && isDashRow(tableLines[i])) i++; const mainData: string[][] = []; for (; i < tableLines.length; i++) { if (tableLines[i].trim() === "") break; // end of table / next #RESOURCE block mainData.push(tableLines[i].split(delimiter).map(v => v.trim())); } if (mainData.length === 0) { throw new Error("No data rows were found."); } const columnIndex = buildColumnIndex(headers); const columnMeta = new Map(); headers.forEach((h, idx) => { const doc = columnDocs.get(h); const unit = units[idx] && units[idx].trim() !== "" ? units[idx].trim() : null; const values = mainData.map(row => row[idx] ?? ""); const type = doc?.type ?? inferType(values); const { min, max } = computeMinMax(values, type); columnMeta.set(h, { type, description: doc?.description ?? null, unit, errorCol: findErrCol(headers, h), min, max, }); }); return { mainData, columnIndex, columnMeta }; } /** * Parses a FITS (Flexible Image Transport System) binary or ascii. * We walk the file's HDUs (header + data blocks), skipping the primary * HDU and any other extensions, until we find one whose * header declares `XTENSION= 'BINTABLE'` or `'TABLE'`. */ function parseFIT(raw: ArrayBuffer): Data { const bytes = new Uint8Array(raw); if (bytesToAscii(bytes.subarray(0, 6)) !== "SIMPLE") { throw new Error("Missing the SIMPLE header card."); } let offset = 0; let table: { xtension: string; header: FitsHeader } | null = null; while (offset < bytes.length) { const header = readFitsHeader(bytes, offset); const xtension = header.cards.get("XTENSION")?.value; if (xtension === "BINTABLE" || xtension === "TABLE") { table = { xtension, header }; break; } offset = header.dataStart + fitsPaddedLength(fitsDataLength(header.cards)); } if (!table) { throw new Error("No table was found in the FITS file."); } return table.xtension === "BINTABLE" ? parseFitsBinTable(bytes, table.header) : parseFitsAsciiTable(bytes, table.header); } /** One BINTABLE column's layout: byte offset within a row, element * type/repeat count, and the bits needed to turn raw bytes into a * display string. */ type BinColumn = { repeat: number; type: string; offset: number; scale: number; zero: number; tnull: string | null; unit: string | null; description: string | null; }; function parseFitsBinTable(bytes: Uint8Array, header: FitsHeader): Data { const cards = header.cards; const nRows = parseInt(cards.get("NAXIS2")?.value ?? "0", 10); const rowWidth = parseInt(cards.get("NAXIS1")?.value ?? "0", 10); const nFields = parseInt(cards.get("TFIELDS")?.value ?? "0", 10); if (nRows === 0 || nFields === 0) { throw new Error("The FITS table extension has no rows or no columns."); } const headers: string[] = []; const columns: BinColumn[] = []; let byteOffset = 0; for (let i = 1; i <= nFields; i++) { const name = cards.get(`TTYPE${i}`)?.value.trim() || `col${i}`; const { repeat, type } = parseBinaryTForm(cards.get(`TFORM${i}`)?.value ?? ""); headers.push(name); columns.push({ repeat, type, offset: byteOffset, scale: cards.has(`TSCAL${i}`) ? parseFloat(cards.get(`TSCAL${i}`)!.value) : 1, zero: cards.has(`TZERO${i}`) ? parseFloat(cards.get(`TZERO${i}`)!.value) : 0, tnull: cards.get(`TNULL${i}`)?.value.trim() ?? null, unit: cards.get(`TUNIT${i}`)?.value.trim() || null, description: cards.get(`TTYPE${i}`)?.comment.trim() || null, }); byteOffset += binaryColumnWidth(repeat, type); } const view = new DataView(bytes.buffer, bytes.byteOffset + header.dataStart, rowWidth * nRows); const mainData: string[][] = []; for (let r = 0; r < nRows; r++) { const rowStart = r * rowWidth; mainData.push(columns.map(col => readBinTableField(view, rowStart + col.offset, col))); } const columnIndex = buildColumnIndex(headers); const columnMeta = new Map(); headers.forEach((h, i) => { const col = columns[i]; const values = mainData.map(row => row[i] ?? ""); // Array columns come out as comma-joined strings, so // only a true scalar numeric column gets marked "number". const isScalarNumber = col.repeat === 1 && ["B", "I", "J", "K", "E", "D"].includes(col.type); const type = isScalarNumber ? "number" : inferType(values); const { min, max } = computeMinMax(values, type); columnMeta.set(h, { type, description: col.description, unit: col.unit, errorCol: findErrCol(headers, h), min, max, }); }); return { mainData, columnIndex, columnMeta }; } /** Decodes one BINTABLE cell. Numeric array columns are * joined with commas into a single string, since `Data.mainData` is * `string[][]`*/ function readBinTableField(view: DataView, offset: number, col: BinColumn): string { const { repeat, type, scale, zero, tnull } = col; if (type === "A") { let s = ""; for (let i = 0; i < repeat; i++) s += String.fromCharCode(view.getUint8(offset + i)); return s.replace(/[\s\0]+$/, ""); // FITS pads short strings with spaces } if (type === "L") { let s = ""; for (let i = 0; i < repeat; i++) { const c = view.getUint8(offset + i); s += c === 84 ? "T" : c === 70 ? "F" : ""; // 'T'/'F'; anything else means "undefined" } return s; } // Bit arrays, complex numbers, and variable-length array descriptors if (type === "X" || type === "C" || type === "M" || type === "P" || type === "Q") return ""; const size = binaryTypeSize(type); const values: string[] = []; for (let i = 0; i < repeat; i++) { const pos = offset + i * size; if (type === "K") { // Kept as an exact string. const rawBig = view.getBigInt64(pos, false); if (tnull !== null && rawBig === BigInt(tnull)) { values.push(""); continue; } values.push(scale === 1 && zero === 0 ? rawBig.toString() : (Number(rawBig) * scale + zero).toString()); continue; } const raw = type === "B" ? view.getUint8(pos) : type === "I" ? view.getInt16(pos, false) : type === "J" ? view.getInt32(pos, false) : type === "E" ? view.getFloat32(pos, false) : view.getFloat64(pos, false); // "D" if (tnull !== null && (type === "B" || type === "I" || type === "J") && raw === Number(tnull)) { values.push(""); continue; } values.push((raw * scale + zero).toString()); } return values.join(","); } /** * Parses an ASCII TABLE extension: rows of fixed-width text. */ function parseFitsAsciiTable(bytes: Uint8Array, header: FitsHeader): Data { const cards = header.cards; const nRows = parseInt(cards.get("NAXIS2")?.value ?? "0", 10); const rowWidth = parseInt(cards.get("NAXIS1")?.value ?? "0", 10); const nFields = parseInt(cards.get("TFIELDS")?.value ?? "0", 10); if (nRows === 0 || nFields === 0) { throw new Error("The FITS ASCII table extension has no rows or no columns."); } type AsciiColumn = { start: number; width: number; numeric: boolean; unit: string | null; description: string | null; tnull: string | null; }; const headers: string[] = []; const columns: AsciiColumn[] = []; for (let i = 1; i <= nFields; i++) { const name = cards.get(`TTYPE${i}`)?.value.trim() || `col${i}`; const tform = cards.get(`TFORM${i}`)?.value.trim() ?? ""; const m = tform.match(/^([IFEDA])(\d+)/i); if (!m) throw new Error(`Unrecognized ASCII-table TFORM code: "${tform}"`); headers.push(name); columns.push({ start: parseInt(cards.get(`TBCOL${i}`)?.value ?? "0", 10) - 1, // TBCOL is 1-indexed width: parseInt(m[2], 10), numeric: m[1].toUpperCase() !== "A", unit: cards.get(`TUNIT${i}`)?.value.trim() || null, description: cards.get(`TTYPE${i}`)?.comment.trim() || null, tnull: cards.get(`TNULL${i}`)?.value.trim() ?? null, }); } const mainData: string[][] = []; for (let r = 0; r < nRows; r++) { const rowStart = header.dataStart + r * rowWidth; const rowText = bytesToAscii(bytes.subarray(rowStart, rowStart + rowWidth)); mainData.push(columns.map(col => { const cell = rowText.slice(col.start, col.start + col.width).trim(); return col.tnull !== null && cell === col.tnull ? "" : cell; })); } const columnIndex = buildColumnIndex(headers); const columnMeta = new Map(); headers.forEach((h, i) => { const col = columns[i]; const values = mainData.map(row => row[i] ?? ""); const type = col.numeric ? "number" : inferType(values); const { min, max } = computeMinMax(values, type); columnMeta.set(h, { type, description: col.description, unit: col.unit, errorCol: findErrCol(headers, h), min, max, }); }); return { mainData, columnIndex, columnMeta }; } /** * Parses a VOTable export from VizieR: XML with a `` of * `` column definitions followed by its data. * Only the first `
` in the file is read. * Can be of type .vot, .b64, .b264, or a VOTable-flavored .tsv */ function parseVOT(raw: string): Data { const tableStart = raw.indexOf(" element was found in the VOTable file."); } const candidates = (["TABLEDATA", "CSV", "BINARY2", "BINARY"] as const) .map(kind => ({ kind, index: raw.indexOf(kind === "TABLEDATA" ? "`, tableStart), })) .filter(c => c.index !== -1) .sort((a, b) => a.index - b.index); const found = candidates[0]; if (!found) { throw new Error("The VOTable has no , , or section - nothing to read rows from."); } const fields = parseVotableFields(raw.slice(tableStart, found.index)); if (fields.length === 0) { throw new Error("No columns were found in the VOTable file."); } const headers = fields.map(f => f.name); const mainData = found.kind === "TABLEDATA" ? parseVotableTabledata(raw, found.index) : found.kind === "CSV" ? parseVotableCsvStream(raw, found.index) : parseVotableBinaryStream(raw, found.index, fields, found.kind === "BINARY2"); if (mainData.length === 0) { throw new Error("No data rows were found in the VOTable file."); } const columnIndex = buildColumnIndex(headers); const columnMeta = new Map(); fields.forEach((f, idx) => { // TABLEDATA and CSV are both plain text, so a nullable // column's sentinel (``) has to be swapped // for "" after the fact. BINARY/BINARY2 nulls are already // resolved to "" while decoding the stream // (readVotableBinaryField), so this only ever does something // on the TABLEDATA/CSV branches. if ((found.kind === "TABLEDATA" || found.kind === "CSV") && f.nullValue !== null) { for (const row of mainData) { if (row[idx] === f.nullValue) row[idx] = ""; } } const values = mainData.map(row => row[idx] ?? ""); const type = f.datatype ? classifyVotableType(f.datatype) : inferType(values); const { min, max } = computeMinMax(values, type); columnMeta.set(f.name, { type, description: f.description, unit: f.unit, errorCol: findErrCol(headers, f.name), min, max, }); }); return { mainData, columnIndex, columnMeta }; } /** Parses the `...` rows of a plain-text * VOTable (VizieR's default `.vot` export). `dataStart` must point at * the `", dataStart); if (dataEnd === -1) { throw new Error("The VOTable's section is missing its closing tag."); } const tableDataXml = raw.slice(dataStart, dataEnd); const mainData: string[][] = []; const rowRe = /([\s\S]*?)<\/TR>/g; let m: RegExpExecArray | null; while ((m = rowRe.exec(tableDataXml)) !== null) { mainData.push(parseVotableRow(m[1])); } return mainData; } /** * Parses the `...` alternate * `` serialization VizieR emits for its "-tsv" VOTable output: * a delimited-text table sitting in a CDATA section instead of * `` rows. Column metadata still comes from the ``s * parsed above, so the `headlines` count of column-name/unit/dashes * rows at the top of the block only needs to be skipped, not read. * * `tagStart` must point at the `", tagStart); const dataEnd = raw.indexOf("", tagStart); if (tagEnd === -1 || dataEnd === -1) { throw new Error("The VOTable's section is missing its closing tag."); } const tag = raw.slice(tagStart, tagEnd + 1); const colsep = getXmlAttr(tag, "colsep") ?? ";"; const headlines = parseInt(getXmlAttr(tag, "headlines") ?? "0", 10) || 0; // Content sits in a CDATA section, so it's literal text - no XML // entity decoding needed (or wanted: CDATA isn't entity-escaped). let body = raw.slice(tagEnd + 1, dataEnd); const cdataMatch = body.match(//); if (cdataMatch) body = cdataMatch[1]; const lines = body.split("\n").filter(l => l.trim().length > 0); return lines.slice(headlines).map(line => line.split(colsep).map(v => v.trim())); } /** One 's byte layout inside a BINARY/BINARY2 stream. */ type VotableBinField = { datatype: string; fixedCount: number; variable: boolean; nullValue: string | null; }; /** * Parses the `` or `` stream of a VizieR VOTable * * `tagStart` must point at the ``/`` tag itself. */ function parseVotableBinaryStream(raw: string, tagStart: number, fields: VotableField[], hasNullFlags: boolean): string[][] { const streamStart = raw.indexOf(" element was found inside the BINARY/BINARY2 section."); } const streamTagEnd = raw.indexOf(">", streamStart); const streamEnd = streamTagEnd === -1 ? -1 : raw.indexOf("", streamTagEnd); if (streamTagEnd === -1 || streamEnd === -1) { throw new Error("The VOTable's element is malformed."); } const bytes = base64ToBytes(raw.slice(streamTagEnd + 1, streamEnd)); const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); const binFields: VotableBinField[] = fields.map(f => { if (!f.datatype) throw new Error(`Field "${f.name}" has no datatype attribute - can't read it from a binary stream.`); const { fixedCount, variable } = parseVotableArraySize(f.arraysize); return { datatype: f.datatype, fixedCount, variable, nullValue: f.nullValue }; }); const flagBytes = Math.ceil(binFields.length / 8); const mainData: string[][] = []; let offset = 0; while (offset < bytes.length) { let nullFlags: boolean[] | null = null; if (hasNullFlags) { nullFlags = binFields.map((_, i) => ((bytes[offset + (i >> 3)] >> (7 - (i % 8))) & 1) === 1); offset += flagBytes; } const row: string[] = []; for (let i = 0; i < binFields.length; i++) { const read = readVotableBinaryField(bytes, view, offset, binFields[i]); offset = read.nextOffset; row.push(nullFlags?.[i] ? "" : read.value); } mainData.push(row); } return mainData; } /** * Decodes one field's value out of a BINARY/BINARY2 stream at * `offset`, returning its display string and the offset right after * it. */ function readVotableBinaryField(bytes: Uint8Array, view: DataView, offset: number, field: VotableBinField): { value: string; nextOffset: number } { let pos = offset; let count = field.fixedCount; if (field.variable) { count = view.getInt32(pos, false); pos += 4; } if (field.datatype === "char") { let s = ""; for (let i = 0; i < count; i++) s += String.fromCharCode(bytes[pos + i]); return { value: s, nextOffset: pos + count }; } if (field.datatype === "unicodeChar") { let s = ""; for (let i = 0; i < count; i++) s += String.fromCharCode(view.getUint16(pos + i * 2, false)); return { value: s, nextOffset: pos + count * 2 }; } if (field.datatype === "boolean") { let s = ""; for (let i = 0; i < count; i++) { const c = String.fromCharCode(bytes[pos + i]); s += (c === "T" || c === "t" || c === "1") ? "T" : (c === "F" || c === "f" || c === "0") ? "F" : ""; } return { value: s, nextOffset: pos + count }; } const size = votableElementSize(field.datatype); const values: string[] = []; for (let i = 0; i < count; i++) { const at = pos + i * size; if (field.datatype === "long") { const raw = view.getBigInt64(at, false); values.push(field.nullValue !== null && raw.toString() === field.nullValue ? "" : raw.toString()); continue; } const raw = field.datatype === "unsignedByte" ? view.getUint8(at) : field.datatype === "short" ? view.getInt16(at, false) : field.datatype === "int" ? view.getInt32(at, false) : field.datatype === "float" ? view.getFloat32(at, false) : view.getFloat64(at, false); // "double" const isNull = Number.isNaN(raw) || (field.nullValue !== null && raw.toString() === field.nullValue); values.push(isNull ? "" : raw.toString()); } return { value: values.join(","), nextOffset: pos + count * size }; } //////////////////////////////////////////////////////////////////////////////////////////////////// /// COLUMN INDEX /////////////////////////////////////////////////////////////////////////////////// /** Maps normalized column names to their index in a CSV row */ export type ColumnIndex = Map; export function getColumnNumber(index: ColumnIndex, names: string[]): Result { for (const name of names) { const i = index.get(name); if (i !== undefined) return ok(i); } return err(new Error("The index for this column wasnt found.")); } //////////////////////////////////////////////////////////////////////////////////////////////////// /// COLUMN METADATA //////////////////////////////////////////////////////////////////////////////// /** Everything we know about a single column of a loaded file. */ type ColumnMeta = { /** Value type, inferred from the file's own data */ type: "number" | "string"; /** From the source file when it carries one, else a prettified `name` */ description: string | null; /** e.g. "mas", "deg". `null` when the file didn't carry one */ unit: string | null; min: number | null; max: number | null; errorCol: number | null; }; export function getMeta(data: Data, names: string[]): Result { for (const name of names) { const meta = data.columnMeta.get(name) if (meta) return ok(meta) } return err(new Error("No metadata found for this column.")) } //////////////////////////////////////////////////////////////////////////////////////////////////// /// GET STAR ///////////////////////////////////////////////////////////////////////////////////// export function findRowNumber(data: Data, header: string, value: string): Result { const colIndexResult = getColumnNumber(data.columnIndex, [header]); if (!colIndexResult.ok) return colIndexResult; const colIndex = unwrap(colIndexResult); const target = value.trim(); for (let i = 0; i < data.mainData.length; i++) { if (data.mainData[i][colIndex] === target) return ok(i); } return err(new Error("The target wasnt found on the data.")); } export function getRow(data: Data, index: number): string[] | null { const r = data.mainData[index]; return r !== undefined ? r : null; } //////////////////////////////////////////////////////////////////////////////////////////////////// /// STAR DATA ////////////////////////////////////////////////////////////////////////////////// /** A row's worth of the fields we usually care about, already typed and * with `null` for anything the source file didn't have. */ export type StarData = { Source: string | null; GMag: string | null; Plx: string | null; Ra: string | null; Dej: string | null; Pm: string | null; }; const FIELD_ALIASES: Record = { Source: ["source", "Source"], GMag: ["Gmag", "phot_g_mean_mag", "G"], Plx: ["Plx", "plx", "parallax"], Ra: ["RA_ICRS", "ra", "RAJ2000"], Dej: ["DE_ICRS", "dej", "DEJ2000"], Pm: ["PM", "pm"], }; function rawFieldValue(data: Data, row: string[], field: keyof StarData): Result { const idxResult = getColumnNumber(data.columnIndex, FIELD_ALIASES[field]); if (!idxResult.ok) return idxResult; const idx = unwrap(idxResult); const idxMetaResult = getMeta(data, FIELD_ALIASES[field]); if (!idxMetaResult.ok) return idxMetaResult; const idxMeta = unwrap(idxMetaResult); const value = row[idx]; if (value === undefined) return err(new Error("The value wasnt found on the data")); if (value.trim() === "") return err(new Error("The value found was empty")); if (!idxMeta!.unit) return ok(value); return ok(value.trim() + " " + idxMeta!.unit) } export function getStarRecord(data: Data, rowIndex: number): Result { const row = data.mainData[rowIndex]; if (!row) return err(new Error("Incorrect Row")); const _rawFieldValue = (field: keyof StarData) => ( unwrapOr(rawFieldValue(data, row, field), null) ) return ok({ Source: _rawFieldValue("Source"), GMag: _rawFieldValue("GMag"), Plx: _rawFieldValue("Plx"), Ra: _rawFieldValue("Ra"), Dej: _rawFieldValue("Dej"), Pm: _rawFieldValue("Pm"), }); } ////////////////////////////////////////////////////////////////////////////////////////////////////