feat(journals): implement bulk Markdown import with multiple parsing strategies
- Added 'ae_journals_parsers.ts' with Standard, Personal Log, and Amazon Vine parsers (ported from Python) - Created 'AeCompModalJournalImport' for file selection, preview, and API submission - Integrated Import button into Journals list view
This commit is contained in:
Scott Idem
214
src/lib/ae_journals/ae_journals_parsers.ts
Normal file
214
src/lib/ae_journals/ae_journals_parsers.ts
Normal file
@@ -0,0 +1,214 @@
|
||||
import { ae_util } from '$lib/ae_utils/ae_utils';
|
||||
import type { key_val } from '$lib/stores/ae_stores';
|
||||
|
||||
export interface AeJournalEntryInput {
|
||||
name: string;
|
||||
content: string;
|
||||
tags: string[];
|
||||
created_on?: string; // ISO string
|
||||
updated_on?: string; // ISO string
|
||||
import_id?: string;
|
||||
external_id?: string;
|
||||
type_code?: string;
|
||||
original_filename: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Standard Parser
|
||||
* - Treats the whole file as one entry.
|
||||
* - First line is title (if it looks like a title).
|
||||
* - Rest is content.
|
||||
*/
|
||||
export async function parse_standard_note(file: File, text: string): Promise<AeJournalEntryInput[]> {
|
||||
const lines = text.split('\n');
|
||||
let name = file.name.replace(/\.md$/i, '').replace(/\.txt$/i, '');
|
||||
let content = text;
|
||||
const tags: string[] = [];
|
||||
|
||||
// Heuristic: If first line is a header, use it as name
|
||||
if (lines.length > 0 && lines[0].startsWith('# ')) {
|
||||
name = lines[0].substring(2).trim();
|
||||
content = lines.slice(1).join('\n').trim();
|
||||
} else if (lines.length > 0 && lines[0].trim().length > 0 && lines[0].trim().length < 60) {
|
||||
// First line is short, treat as title if it doesn't look like frontmatter
|
||||
if (lines[0].trim() !== '---') {
|
||||
name = lines[0].trim();
|
||||
content = lines.slice(1).join('\n').trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Basic Frontmatter check (YAML style)
|
||||
if (lines[0]?.trim() === '---') {
|
||||
const endFrontmatter = lines.indexOf('---', 1);
|
||||
if (endFrontmatter > -1) {
|
||||
const frontmatter = lines.slice(1, endFrontmatter);
|
||||
content = lines.slice(endFrontmatter + 1).join('\n').trim();
|
||||
|
||||
// Extract tags or title from frontmatter (very basic parsing)
|
||||
frontmatter.forEach(line => {
|
||||
if (line.startsWith('title:')) name = line.substring(6).trim().replace(/^['"]|['"]$/g, '');
|
||||
if (line.startsWith('tags:')) {
|
||||
// This is brittle, assumes inline tags like [a, b] or comma separated
|
||||
const tagPart = line.substring(5).trim();
|
||||
if (tagPart.startsWith('[') && tagPart.endsWith(']')) {
|
||||
tagPart.substring(1, tagPart.length - 1).split(',').forEach(t => tags.push(t.trim()));
|
||||
} else {
|
||||
tagPart.split(',').forEach(t => tags.push(t.trim()));
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const lastModified = new Date(file.lastModified).toISOString();
|
||||
|
||||
return [{
|
||||
name,
|
||||
content,
|
||||
tags,
|
||||
updated_on: lastModified,
|
||||
created_on: lastModified, // We don't really know creation time from File object usually
|
||||
original_filename: file.name,
|
||||
type_code: 'note'
|
||||
}];
|
||||
}
|
||||
|
||||
/**
|
||||
* Personal Log Parser
|
||||
* - Splits file by dates: `## YYYY-MM-DD`
|
||||
*/
|
||||
export async function parse_personal_log(file: File, text: string): Promise<AeJournalEntryInput[]> {
|
||||
const entries: AeJournalEntryInput[] = [];
|
||||
const dateRegex = /^##\s+(\d{4}-\d{2}-\d{2})(.*)$/;
|
||||
|
||||
const lines = text.split('\n');
|
||||
let currentEntry: Partial<AeJournalEntryInput> | null = null;
|
||||
let currentContent: string[] = [];
|
||||
|
||||
// Check if the whole file is just one entry (no date headers)
|
||||
if (!lines.some(l => dateRegex.test(l))) {
|
||||
return parse_standard_note(file, text);
|
||||
}
|
||||
|
||||
const fileBaseName = file.name.replace(/\.md$/i, '').replace(/\.txt$/i, '');
|
||||
|
||||
for (const line of lines) {
|
||||
const match = line.match(dateRegex);
|
||||
if (match) {
|
||||
// Save previous entry
|
||||
if (currentEntry) {
|
||||
currentEntry.content = currentContent.join('\n').trim();
|
||||
entries.push(currentEntry as AeJournalEntryInput);
|
||||
}
|
||||
|
||||
// Start new entry
|
||||
const dateStr = match[1];
|
||||
const extraTitle = match[2].trim();
|
||||
|
||||
currentEntry = {
|
||||
name: extraTitle ? `${dateStr} - ${extraTitle}` : `${fileBaseName} - ${dateStr}`,
|
||||
created_on: `${dateStr}T12:00:00`, // Noon on that day
|
||||
updated_on: new Date(file.lastModified).toISOString(),
|
||||
tags: ['log'],
|
||||
type_code: 'log',
|
||||
original_filename: file.name,
|
||||
// Reconstruct the header as part of content? Or just skip it?
|
||||
// Python parser added it back: `## {date_str}\n\n{body}`
|
||||
// Let's add it back for context.
|
||||
};
|
||||
currentContent = [`## ${dateStr} ${extraTitle}`];
|
||||
} else {
|
||||
if (currentEntry) {
|
||||
currentContent.push(line);
|
||||
} else {
|
||||
// Preamble before first date header? Ignore or treat as separate?
|
||||
// Ignoring for now or could be a "Header" entry.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Push last entry
|
||||
if (currentEntry) {
|
||||
currentEntry.content = currentContent.join('\n').trim();
|
||||
entries.push(currentEntry as AeJournalEntryInput);
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
/**
|
||||
* Amazon Vine Review Parser (from Python logic)
|
||||
* - Splits by `## Product Name`
|
||||
* - Looks for URL and `### Review Title`
|
||||
*/
|
||||
export async function parse_amazon_vine(file: File, text: string): Promise<AeJournalEntryInput[]> {
|
||||
// Split by `\n## ` but we need to keep the delimiter or reconstruct
|
||||
// JS split doesn't keep delimiter nicely unless captured.
|
||||
// Let's iterate lines.
|
||||
const entries: AeJournalEntryInput[] = [];
|
||||
const productHeaderRegex = /^##\s+(.+)$/;
|
||||
|
||||
const lines = text.split('\n');
|
||||
let currentEntry: any = null;
|
||||
let currentBody: string[] = [];
|
||||
|
||||
for (const line of lines) {
|
||||
const match = line.match(productHeaderRegex);
|
||||
if (match) {
|
||||
if (currentEntry) {
|
||||
entries.push(format_vine_entry(currentEntry, currentBody, file));
|
||||
}
|
||||
currentEntry = { productName: match[1].trim() };
|
||||
currentBody = [];
|
||||
} else {
|
||||
if (currentEntry) {
|
||||
currentBody.push(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (currentEntry) {
|
||||
entries.push(format_vine_entry(currentEntry, currentBody, file));
|
||||
}
|
||||
|
||||
return entries;
|
||||
}
|
||||
|
||||
function format_vine_entry(entry: any, bodyLines: string[], file: File): AeJournalEntryInput {
|
||||
let url = '';
|
||||
let reviewTitle = '';
|
||||
const cleanBody: string[] = [];
|
||||
|
||||
for (const line of bodyLines) {
|
||||
const trimmed = line.trim();
|
||||
if (!url && trimmed.startsWith('* http')) {
|
||||
url = trimmed.replace(/^\*\s+/, '').split(' ')[0];
|
||||
continue;
|
||||
}
|
||||
if (!reviewTitle && trimmed.startsWith('### ')) {
|
||||
reviewTitle = trimmed.substring(4).trim();
|
||||
continue;
|
||||
}
|
||||
cleanBody.push(line);
|
||||
}
|
||||
|
||||
let content = '';
|
||||
if (reviewTitle) content += `# ${reviewTitle}\n\n`;
|
||||
content += cleanBody.join('\n').trim();
|
||||
if (url) content += `\n\n**Product Link:** ${url}`;
|
||||
|
||||
return {
|
||||
name: entry.productName,
|
||||
content: content,
|
||||
tags: ['amazon', 'vine', 'review'],
|
||||
created_on: new Date(file.lastModified).toISOString(),
|
||||
updated_on: new Date(file.lastModified).toISOString(),
|
||||
original_filename: file.name,
|
||||
type_code: 'review'
|
||||
};
|
||||
}
|
||||
|
||||
export const PARSERS = {
|
||||
standard: parse_standard_note,
|
||||
personal_log: parse_personal_log,
|
||||
amazon_vine: parse_amazon_vine
|
||||
};
|
||||
@@ -8,7 +8,7 @@
|
||||
import { goto } from '$app/navigation';
|
||||
|
||||
// *** Import other supporting libraries
|
||||
import { BookPlus, FolderPlus, Library, Loader, SquareLibrary, Wrench } from '@lucide/svelte';
|
||||
import { BookPlus, FolderPlus, Library, Loader, SquareLibrary, Wrench, FileUp } from '@lucide/svelte';
|
||||
import { liveQuery } from 'dexie';
|
||||
import { Modal } from 'flowbite-svelte';
|
||||
|
||||
@@ -33,10 +33,19 @@
|
||||
import Modal_journals_cfg from './modal_journals_config.svelte';
|
||||
import Journal_obj_li from './ae_comp__journal_obj_li.svelte';
|
||||
import AeCompJournalEntryQuickAdd from './ae_comp__journal_entry_quick_add.svelte';
|
||||
import AeCompModalJournalImport from './ae_comp__modal_journal_import.svelte';
|
||||
|
||||
// import Element_data_store from '$lib/element_data_store_v2.svelte';
|
||||
|
||||
let ae_acct = data[$slct.account_id];
|
||||
let show_import_modal = $state(false);
|
||||
|
||||
function handle_import_complete() {
|
||||
// Trigger a refresh of the journal list?
|
||||
// Usually liveQuery handles it if data changed in IDB.
|
||||
// But if we just created entries, we might want to refresh the selected journal view if active.
|
||||
// $journals_trig.journal_entry_li = true;
|
||||
}
|
||||
// $journals_slct.journal_obj = ae_acct.slct.journal_obj;
|
||||
// $journals_slct.journal_obj_li = ae_acct.slct.journal_obj_li;
|
||||
|
||||
@@ -204,6 +213,24 @@
|
||||
<span class="hidden md:inline"> New Journal </span>
|
||||
</button>
|
||||
|
||||
<!-- Import Entries button -->
|
||||
<button
|
||||
type="button"
|
||||
class="
|
||||
btn btn-sm
|
||||
preset-tonal-secondary border border-secondary-500
|
||||
hover:preset-filled-secondary-500
|
||||
transition
|
||||
"
|
||||
onclick={() => {
|
||||
show_import_modal = true;
|
||||
}}
|
||||
title="Import entries from Markdown files"
|
||||
>
|
||||
<FileUp class="mx-1" />
|
||||
<span class="hidden md:inline"> Import </span>
|
||||
</button>
|
||||
|
||||
<!-- Show Journals Config button -->
|
||||
<button
|
||||
type="button"
|
||||
@@ -326,5 +353,11 @@
|
||||
<Modal_journals_cfg show={$journals_sess.show__modal__journals_config} />
|
||||
{/if}
|
||||
|
||||
<AeCompModalJournalImport
|
||||
bind:open={show_import_modal}
|
||||
onClose={() => (show_import_modal = false)}
|
||||
onImportComplete={handle_import_complete}
|
||||
/>
|
||||
|
||||
<style lang="postcss">
|
||||
</style>
|
||||
|
||||
195
src/routes/journals/ae_comp__modal_journal_import.svelte
Normal file
195
src/routes/journals/ae_comp__modal_journal_import.svelte
Normal file
@@ -0,0 +1,195 @@
|
||||
<script lang="ts">
|
||||
import { Modal } from 'flowbite-svelte';
|
||||
import { Upload, FileText, AlertCircle, Check, X, RefreshCw } from '@lucide/svelte';
|
||||
import { PARSERS, type AeJournalEntryInput } from '$lib/ae_journals/ae_journals_parsers';
|
||||
import { journals_func } from '$lib/ae_journals/ae_journals_functions';
|
||||
import { ae_api } from '$lib/stores/ae_stores';
|
||||
import { journals_slct } from '$lib/ae_journals/ae_journals_stores';
|
||||
|
||||
interface Props {
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
onImportComplete: () => void;
|
||||
}
|
||||
|
||||
let { open = $bindable(false), onClose, onImportComplete }: Props = $props();
|
||||
|
||||
let files: FileList | null = $state(null);
|
||||
let selected_parser: keyof typeof PARSERS = $state('standard');
|
||||
let parsed_entries: AeJournalEntryInput[] = $state([]);
|
||||
let is_parsing = $state(false);
|
||||
let is_importing = $state(false);
|
||||
let import_log: string[] = $state([]);
|
||||
|
||||
// Watch for file selection or parser change to trigger parsing
|
||||
$effect(() => {
|
||||
if (files && files.length > 0) {
|
||||
parse_files();
|
||||
}
|
||||
});
|
||||
|
||||
async function parse_files() {
|
||||
if (!files) return;
|
||||
is_parsing = true;
|
||||
parsed_entries = [];
|
||||
|
||||
const parser = PARSERS[selected_parser];
|
||||
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
try {
|
||||
const text = await file.text();
|
||||
const entries = await parser(file, text);
|
||||
parsed_entries = [...parsed_entries, ...entries];
|
||||
} catch (err) {
|
||||
console.error(`Error parsing ${file.name}:`, err);
|
||||
}
|
||||
}
|
||||
is_parsing = false;
|
||||
}
|
||||
|
||||
async function handle_import() {
|
||||
if (parsed_entries.length === 0) return;
|
||||
is_importing = true;
|
||||
import_log = [];
|
||||
|
||||
const journal_id = $journals_slct.journal_id;
|
||||
if (!journal_id) {
|
||||
alert("No target journal selected. Please select a journal in the background first.");
|
||||
is_importing = false;
|
||||
return;
|
||||
}
|
||||
|
||||
let success_count = 0;
|
||||
|
||||
for (const entry of parsed_entries) {
|
||||
try {
|
||||
// Construct payload
|
||||
const data_kv = {
|
||||
name: entry.name,
|
||||
content: entry.content,
|
||||
tags: entry.tags.join(', '),
|
||||
type_code: entry.type_code || 'note',
|
||||
created_on: entry.created_on,
|
||||
updated_on: entry.updated_on
|
||||
};
|
||||
|
||||
const res = await journals_func.create_ae_obj__journal_entry({
|
||||
api_cfg: $ae_api,
|
||||
journal_id: journal_id,
|
||||
data_kv: data_kv,
|
||||
log_lvl: 0
|
||||
});
|
||||
|
||||
if (res) {
|
||||
import_log.push(`✅ Imported: ${entry.name}`);
|
||||
success_count++;
|
||||
} else {
|
||||
import_log.push(`❌ Failed: ${entry.name}`);
|
||||
}
|
||||
} catch (err) {
|
||||
import_log.push(`❌ Error: ${entry.name} - ${err}`);
|
||||
}
|
||||
}
|
||||
|
||||
is_importing = false;
|
||||
alert(`Import complete! ${success_count}/${parsed_entries.length} imported.`);
|
||||
onImportComplete();
|
||||
open = false;
|
||||
}
|
||||
</script>
|
||||
|
||||
<Modal
|
||||
title="Import Journal Entries"
|
||||
bind:open={open}
|
||||
autoclose={false}
|
||||
size="xl"
|
||||
class="w-full"
|
||||
>
|
||||
<div class="space-y-4">
|
||||
<!-- Configuration -->
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="label">
|
||||
<span>Parser Strategy</span>
|
||||
<select class="select" bind:value={selected_parser}>
|
||||
<option value="standard">Standard (1 File = 1 Entry)</option>
|
||||
<option value="personal_log">Personal Log (Split by Date)</option>
|
||||
<option value="amazon_vine">Amazon Vine Reviews</option>
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
<div>
|
||||
<label class="label">
|
||||
<span>Select Files</span>
|
||||
<input
|
||||
type="file"
|
||||
class="file-input w-full"
|
||||
multiple
|
||||
accept=".md,.txt"
|
||||
onchange={(e) => files = e.currentTarget.files}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Preview -->
|
||||
<div class="border rounded-lg p-2 bg-gray-50 dark:bg-gray-900 max-h-64 overflow-y-auto">
|
||||
<h4 class="font-bold mb-2 flex justify-between">
|
||||
<span>Preview ({parsed_entries.length} entries)</span>
|
||||
{#if is_parsing}
|
||||
<RefreshCw class="animate-spin" />
|
||||
{/if}
|
||||
</h4>
|
||||
|
||||
{#if parsed_entries.length > 0}
|
||||
<table class="table table-compact w-full text-xs">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Date</th>
|
||||
<th>Tags</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{#each parsed_entries as entry}
|
||||
<tr>
|
||||
<td class="truncate max-w-[200px]" title={entry.name}>{entry.name}</td>
|
||||
<td>{entry.created_on?.substring(0,10)}</td>
|
||||
<td>{entry.tags.join(', ')}</td>
|
||||
</tr>
|
||||
{/each}
|
||||
</tbody>
|
||||
</table>
|
||||
{:else if files && files.length > 0 && !is_parsing}
|
||||
<div class="text-center text-gray-500 py-4">No entries found in selected files.</div>
|
||||
{:else if !files}
|
||||
<div class="text-center text-gray-500 py-4">Select files to preview import.</div>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Import Log -->
|
||||
{#if import_log.length > 0}
|
||||
<div class="bg-black text-green-400 p-2 rounded text-xs font-mono max-h-32 overflow-y-auto">
|
||||
{#each import_log as log}
|
||||
<div>{log}</div>
|
||||
{/each}
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<div class="modal-action">
|
||||
<button class="btn preset-tonal-secondary" onclick={onClose}>Cancel</button>
|
||||
<button
|
||||
class="btn preset-filled-primary"
|
||||
disabled={parsed_entries.length === 0 || is_importing}
|
||||
onclick={handle_import}
|
||||
>
|
||||
{#if is_importing}
|
||||
Importing...
|
||||
{:else}
|
||||
Import {parsed_entries.length} Entries
|
||||
{/if}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
Reference in New Issue
Block a user