feat(journals): implement bulk Markdown import with multiple parsing strategies
- Added 'ae_journals_parsers.ts' with Standard, Personal Log, and Amazon Vine parsers (ported from Python) - Created 'AeCompModalJournalImport' for file selection, preview, and API submission - Integrated Import button into Journals list view
This commit is contained in:
214
src/lib/ae_journals/ae_journals_parsers.ts
Normal file
214
src/lib/ae_journals/ae_journals_parsers.ts
Normal file
@@ -0,0 +1,214 @@
|
|||||||
|
import { ae_util } from '$lib/ae_utils/ae_utils';
|
||||||
|
import type { key_val } from '$lib/stores/ae_stores';
|
||||||
|
|
||||||
|
export interface AeJournalEntryInput {
|
||||||
|
name: string;
|
||||||
|
content: string;
|
||||||
|
tags: string[];
|
||||||
|
created_on?: string; // ISO string
|
||||||
|
updated_on?: string; // ISO string
|
||||||
|
import_id?: string;
|
||||||
|
external_id?: string;
|
||||||
|
type_code?: string;
|
||||||
|
original_filename: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Standard Parser
|
||||||
|
* - Treats the whole file as one entry.
|
||||||
|
* - First line is title (if it looks like a title).
|
||||||
|
* - Rest is content.
|
||||||
|
*/
|
||||||
|
export async function parse_standard_note(file: File, text: string): Promise<AeJournalEntryInput[]> {
|
||||||
|
const lines = text.split('\n');
|
||||||
|
let name = file.name.replace(/\.md$/i, '').replace(/\.txt$/i, '');
|
||||||
|
let content = text;
|
||||||
|
const tags: string[] = [];
|
||||||
|
|
||||||
|
// Heuristic: If first line is a header, use it as name
|
||||||
|
if (lines.length > 0 && lines[0].startsWith('# ')) {
|
||||||
|
name = lines[0].substring(2).trim();
|
||||||
|
content = lines.slice(1).join('\n').trim();
|
||||||
|
} else if (lines.length > 0 && lines[0].trim().length > 0 && lines[0].trim().length < 60) {
|
||||||
|
// First line is short, treat as title if it doesn't look like frontmatter
|
||||||
|
if (lines[0].trim() !== '---') {
|
||||||
|
name = lines[0].trim();
|
||||||
|
content = lines.slice(1).join('\n').trim();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Basic Frontmatter check (YAML style)
|
||||||
|
if (lines[0]?.trim() === '---') {
|
||||||
|
const endFrontmatter = lines.indexOf('---', 1);
|
||||||
|
if (endFrontmatter > -1) {
|
||||||
|
const frontmatter = lines.slice(1, endFrontmatter);
|
||||||
|
content = lines.slice(endFrontmatter + 1).join('\n').trim();
|
||||||
|
|
||||||
|
// Extract tags or title from frontmatter (very basic parsing)
|
||||||
|
frontmatter.forEach(line => {
|
||||||
|
if (line.startsWith('title:')) name = line.substring(6).trim().replace(/^['"]|['"]$/g, '');
|
||||||
|
if (line.startsWith('tags:')) {
|
||||||
|
// This is brittle, assumes inline tags like [a, b] or comma separated
|
||||||
|
const tagPart = line.substring(5).trim();
|
||||||
|
if (tagPart.startsWith('[') && tagPart.endsWith(']')) {
|
||||||
|
tagPart.substring(1, tagPart.length - 1).split(',').forEach(t => tags.push(t.trim()));
|
||||||
|
} else {
|
||||||
|
tagPart.split(',').forEach(t => tags.push(t.trim()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const lastModified = new Date(file.lastModified).toISOString();
|
||||||
|
|
||||||
|
return [{
|
||||||
|
name,
|
||||||
|
content,
|
||||||
|
tags,
|
||||||
|
updated_on: lastModified,
|
||||||
|
created_on: lastModified, // We don't really know creation time from File object usually
|
||||||
|
original_filename: file.name,
|
||||||
|
type_code: 'note'
|
||||||
|
}];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Personal Log Parser
|
||||||
|
* - Splits file by dates: `## YYYY-MM-DD`
|
||||||
|
*/
|
||||||
|
export async function parse_personal_log(file: File, text: string): Promise<AeJournalEntryInput[]> {
|
||||||
|
const entries: AeJournalEntryInput[] = [];
|
||||||
|
const dateRegex = /^##\s+(\d{4}-\d{2}-\d{2})(.*)$/;
|
||||||
|
|
||||||
|
const lines = text.split('\n');
|
||||||
|
let currentEntry: Partial<AeJournalEntryInput> | null = null;
|
||||||
|
let currentContent: string[] = [];
|
||||||
|
|
||||||
|
// Check if the whole file is just one entry (no date headers)
|
||||||
|
if (!lines.some(l => dateRegex.test(l))) {
|
||||||
|
return parse_standard_note(file, text);
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileBaseName = file.name.replace(/\.md$/i, '').replace(/\.txt$/i, '');
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
const match = line.match(dateRegex);
|
||||||
|
if (match) {
|
||||||
|
// Save previous entry
|
||||||
|
if (currentEntry) {
|
||||||
|
currentEntry.content = currentContent.join('\n').trim();
|
||||||
|
entries.push(currentEntry as AeJournalEntryInput);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start new entry
|
||||||
|
const dateStr = match[1];
|
||||||
|
const extraTitle = match[2].trim();
|
||||||
|
|
||||||
|
currentEntry = {
|
||||||
|
name: extraTitle ? `${dateStr} - ${extraTitle}` : `${fileBaseName} - ${dateStr}`,
|
||||||
|
created_on: `${dateStr}T12:00:00`, // Noon on that day
|
||||||
|
updated_on: new Date(file.lastModified).toISOString(),
|
||||||
|
tags: ['log'],
|
||||||
|
type_code: 'log',
|
||||||
|
original_filename: file.name,
|
||||||
|
// Reconstruct the header as part of content? Or just skip it?
|
||||||
|
// Python parser added it back: `## {date_str}\n\n{body}`
|
||||||
|
// Let's add it back for context.
|
||||||
|
};
|
||||||
|
currentContent = [`## ${dateStr} ${extraTitle}`];
|
||||||
|
} else {
|
||||||
|
if (currentEntry) {
|
||||||
|
currentContent.push(line);
|
||||||
|
} else {
|
||||||
|
// Preamble before first date header? Ignore or treat as separate?
|
||||||
|
// Ignoring for now or could be a "Header" entry.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Push last entry
|
||||||
|
if (currentEntry) {
|
||||||
|
currentEntry.content = currentContent.join('\n').trim();
|
||||||
|
entries.push(currentEntry as AeJournalEntryInput);
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Amazon Vine Review Parser (from Python logic)
|
||||||
|
* - Splits by `## Product Name`
|
||||||
|
* - Looks for URL and `### Review Title`
|
||||||
|
*/
|
||||||
|
export async function parse_amazon_vine(file: File, text: string): Promise<AeJournalEntryInput[]> {
|
||||||
|
// Split by `\n## ` but we need to keep the delimiter or reconstruct
|
||||||
|
// JS split doesn't keep delimiter nicely unless captured.
|
||||||
|
// Let's iterate lines.
|
||||||
|
const entries: AeJournalEntryInput[] = [];
|
||||||
|
const productHeaderRegex = /^##\s+(.+)$/;
|
||||||
|
|
||||||
|
const lines = text.split('\n');
|
||||||
|
let currentEntry: any = null;
|
||||||
|
let currentBody: string[] = [];
|
||||||
|
|
||||||
|
for (const line of lines) {
|
||||||
|
const match = line.match(productHeaderRegex);
|
||||||
|
if (match) {
|
||||||
|
if (currentEntry) {
|
||||||
|
entries.push(format_vine_entry(currentEntry, currentBody, file));
|
||||||
|
}
|
||||||
|
currentEntry = { productName: match[1].trim() };
|
||||||
|
currentBody = [];
|
||||||
|
} else {
|
||||||
|
if (currentEntry) {
|
||||||
|
currentBody.push(line);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (currentEntry) {
|
||||||
|
entries.push(format_vine_entry(currentEntry, currentBody, file));
|
||||||
|
}
|
||||||
|
|
||||||
|
return entries;
|
||||||
|
}
|
||||||
|
|
||||||
|
function format_vine_entry(entry: any, bodyLines: string[], file: File): AeJournalEntryInput {
|
||||||
|
let url = '';
|
||||||
|
let reviewTitle = '';
|
||||||
|
const cleanBody: string[] = [];
|
||||||
|
|
||||||
|
for (const line of bodyLines) {
|
||||||
|
const trimmed = line.trim();
|
||||||
|
if (!url && trimmed.startsWith('* http')) {
|
||||||
|
url = trimmed.replace(/^\*\s+/, '').split(' ')[0];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if (!reviewTitle && trimmed.startsWith('### ')) {
|
||||||
|
reviewTitle = trimmed.substring(4).trim();
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
cleanBody.push(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
let content = '';
|
||||||
|
if (reviewTitle) content += `# ${reviewTitle}\n\n`;
|
||||||
|
content += cleanBody.join('\n').trim();
|
||||||
|
if (url) content += `\n\n**Product Link:** ${url}`;
|
||||||
|
|
||||||
|
return {
|
||||||
|
name: entry.productName,
|
||||||
|
content: content,
|
||||||
|
tags: ['amazon', 'vine', 'review'],
|
||||||
|
created_on: new Date(file.lastModified).toISOString(),
|
||||||
|
updated_on: new Date(file.lastModified).toISOString(),
|
||||||
|
original_filename: file.name,
|
||||||
|
type_code: 'review'
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const PARSERS = {
|
||||||
|
standard: parse_standard_note,
|
||||||
|
personal_log: parse_personal_log,
|
||||||
|
amazon_vine: parse_amazon_vine
|
||||||
|
};
|
||||||
@@ -8,7 +8,7 @@
|
|||||||
import { goto } from '$app/navigation';
|
import { goto } from '$app/navigation';
|
||||||
|
|
||||||
// *** Import other supporting libraries
|
// *** Import other supporting libraries
|
||||||
import { BookPlus, FolderPlus, Library, Loader, SquareLibrary, Wrench } from '@lucide/svelte';
|
import { BookPlus, FolderPlus, Library, Loader, SquareLibrary, Wrench, FileUp } from '@lucide/svelte';
|
||||||
import { liveQuery } from 'dexie';
|
import { liveQuery } from 'dexie';
|
||||||
import { Modal } from 'flowbite-svelte';
|
import { Modal } from 'flowbite-svelte';
|
||||||
|
|
||||||
@@ -33,10 +33,19 @@
|
|||||||
import Modal_journals_cfg from './modal_journals_config.svelte';
|
import Modal_journals_cfg from './modal_journals_config.svelte';
|
||||||
import Journal_obj_li from './ae_comp__journal_obj_li.svelte';
|
import Journal_obj_li from './ae_comp__journal_obj_li.svelte';
|
||||||
import AeCompJournalEntryQuickAdd from './ae_comp__journal_entry_quick_add.svelte';
|
import AeCompJournalEntryQuickAdd from './ae_comp__journal_entry_quick_add.svelte';
|
||||||
|
import AeCompModalJournalImport from './ae_comp__modal_journal_import.svelte';
|
||||||
|
|
||||||
// import Element_data_store from '$lib/element_data_store_v2.svelte';
|
// import Element_data_store from '$lib/element_data_store_v2.svelte';
|
||||||
|
|
||||||
let ae_acct = data[$slct.account_id];
|
let ae_acct = data[$slct.account_id];
|
||||||
|
let show_import_modal = $state(false);
|
||||||
|
|
||||||
|
function handle_import_complete() {
|
||||||
|
// Trigger a refresh of the journal list?
|
||||||
|
// Usually liveQuery handles it if data changed in IDB.
|
||||||
|
// But if we just created entries, we might want to refresh the selected journal view if active.
|
||||||
|
// $journals_trig.journal_entry_li = true;
|
||||||
|
}
|
||||||
// $journals_slct.journal_obj = ae_acct.slct.journal_obj;
|
// $journals_slct.journal_obj = ae_acct.slct.journal_obj;
|
||||||
// $journals_slct.journal_obj_li = ae_acct.slct.journal_obj_li;
|
// $journals_slct.journal_obj_li = ae_acct.slct.journal_obj_li;
|
||||||
|
|
||||||
@@ -204,6 +213,24 @@
|
|||||||
<span class="hidden md:inline"> New Journal </span>
|
<span class="hidden md:inline"> New Journal </span>
|
||||||
</button>
|
</button>
|
||||||
|
|
||||||
|
<!-- Import Entries button -->
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
class="
|
||||||
|
btn btn-sm
|
||||||
|
preset-tonal-secondary border border-secondary-500
|
||||||
|
hover:preset-filled-secondary-500
|
||||||
|
transition
|
||||||
|
"
|
||||||
|
onclick={() => {
|
||||||
|
show_import_modal = true;
|
||||||
|
}}
|
||||||
|
title="Import entries from Markdown files"
|
||||||
|
>
|
||||||
|
<FileUp class="mx-1" />
|
||||||
|
<span class="hidden md:inline"> Import </span>
|
||||||
|
</button>
|
||||||
|
|
||||||
<!-- Show Journals Config button -->
|
<!-- Show Journals Config button -->
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
@@ -326,5 +353,11 @@
|
|||||||
<Modal_journals_cfg show={$journals_sess.show__modal__journals_config} />
|
<Modal_journals_cfg show={$journals_sess.show__modal__journals_config} />
|
||||||
{/if}
|
{/if}
|
||||||
|
|
||||||
|
<AeCompModalJournalImport
|
||||||
|
bind:open={show_import_modal}
|
||||||
|
onClose={() => (show_import_modal = false)}
|
||||||
|
onImportComplete={handle_import_complete}
|
||||||
|
/>
|
||||||
|
|
||||||
<style lang="postcss">
|
<style lang="postcss">
|
||||||
</style>
|
</style>
|
||||||
|
|||||||
195
src/routes/journals/ae_comp__modal_journal_import.svelte
Normal file
195
src/routes/journals/ae_comp__modal_journal_import.svelte
Normal file
@@ -0,0 +1,195 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import { Modal } from 'flowbite-svelte';
|
||||||
|
import { Upload, FileText, AlertCircle, Check, X, RefreshCw } from '@lucide/svelte';
|
||||||
|
import { PARSERS, type AeJournalEntryInput } from '$lib/ae_journals/ae_journals_parsers';
|
||||||
|
import { journals_func } from '$lib/ae_journals/ae_journals_functions';
|
||||||
|
import { ae_api } from '$lib/stores/ae_stores';
|
||||||
|
import { journals_slct } from '$lib/ae_journals/ae_journals_stores';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
open: boolean;
|
||||||
|
onClose: () => void;
|
||||||
|
onImportComplete: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
let { open = $bindable(false), onClose, onImportComplete }: Props = $props();
|
||||||
|
|
||||||
|
let files: FileList | null = $state(null);
|
||||||
|
let selected_parser: keyof typeof PARSERS = $state('standard');
|
||||||
|
let parsed_entries: AeJournalEntryInput[] = $state([]);
|
||||||
|
let is_parsing = $state(false);
|
||||||
|
let is_importing = $state(false);
|
||||||
|
let import_log: string[] = $state([]);
|
||||||
|
|
||||||
|
// Watch for file selection or parser change to trigger parsing
|
||||||
|
$effect(() => {
|
||||||
|
if (files && files.length > 0) {
|
||||||
|
parse_files();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
async function parse_files() {
|
||||||
|
if (!files) return;
|
||||||
|
is_parsing = true;
|
||||||
|
parsed_entries = [];
|
||||||
|
|
||||||
|
const parser = PARSERS[selected_parser];
|
||||||
|
|
||||||
|
for (let i = 0; i < files.length; i++) {
|
||||||
|
const file = files[i];
|
||||||
|
try {
|
||||||
|
const text = await file.text();
|
||||||
|
const entries = await parser(file, text);
|
||||||
|
parsed_entries = [...parsed_entries, ...entries];
|
||||||
|
} catch (err) {
|
||||||
|
console.error(`Error parsing ${file.name}:`, err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
is_parsing = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handle_import() {
|
||||||
|
if (parsed_entries.length === 0) return;
|
||||||
|
is_importing = true;
|
||||||
|
import_log = [];
|
||||||
|
|
||||||
|
const journal_id = $journals_slct.journal_id;
|
||||||
|
if (!journal_id) {
|
||||||
|
alert("No target journal selected. Please select a journal in the background first.");
|
||||||
|
is_importing = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let success_count = 0;
|
||||||
|
|
||||||
|
for (const entry of parsed_entries) {
|
||||||
|
try {
|
||||||
|
// Construct payload
|
||||||
|
const data_kv = {
|
||||||
|
name: entry.name,
|
||||||
|
content: entry.content,
|
||||||
|
tags: entry.tags.join(', '),
|
||||||
|
type_code: entry.type_code || 'note',
|
||||||
|
created_on: entry.created_on,
|
||||||
|
updated_on: entry.updated_on
|
||||||
|
};
|
||||||
|
|
||||||
|
const res = await journals_func.create_ae_obj__journal_entry({
|
||||||
|
api_cfg: $ae_api,
|
||||||
|
journal_id: journal_id,
|
||||||
|
data_kv: data_kv,
|
||||||
|
log_lvl: 0
|
||||||
|
});
|
||||||
|
|
||||||
|
if (res) {
|
||||||
|
import_log.push(`✅ Imported: ${entry.name}`);
|
||||||
|
success_count++;
|
||||||
|
} else {
|
||||||
|
import_log.push(`❌ Failed: ${entry.name}`);
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
import_log.push(`❌ Error: ${entry.name} - ${err}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
is_importing = false;
|
||||||
|
alert(`Import complete! ${success_count}/${parsed_entries.length} imported.`);
|
||||||
|
onImportComplete();
|
||||||
|
open = false;
|
||||||
|
}
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<Modal
|
||||||
|
title="Import Journal Entries"
|
||||||
|
bind:open={open}
|
||||||
|
autoclose={false}
|
||||||
|
size="xl"
|
||||||
|
class="w-full"
|
||||||
|
>
|
||||||
|
<div class="space-y-4">
|
||||||
|
<!-- Configuration -->
|
||||||
|
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||||
|
<div>
|
||||||
|
<label class="label">
|
||||||
|
<span>Parser Strategy</span>
|
||||||
|
<select class="select" bind:value={selected_parser}>
|
||||||
|
<option value="standard">Standard (1 File = 1 Entry)</option>
|
||||||
|
<option value="personal_log">Personal Log (Split by Date)</option>
|
||||||
|
<option value="amazon_vine">Amazon Vine Reviews</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
<div>
|
||||||
|
<label class="label">
|
||||||
|
<span>Select Files</span>
|
||||||
|
<input
|
||||||
|
type="file"
|
||||||
|
class="file-input w-full"
|
||||||
|
multiple
|
||||||
|
accept=".md,.txt"
|
||||||
|
onchange={(e) => files = e.currentTarget.files}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Preview -->
|
||||||
|
<div class="border rounded-lg p-2 bg-gray-50 dark:bg-gray-900 max-h-64 overflow-y-auto">
|
||||||
|
<h4 class="font-bold mb-2 flex justify-between">
|
||||||
|
<span>Preview ({parsed_entries.length} entries)</span>
|
||||||
|
{#if is_parsing}
|
||||||
|
<RefreshCw class="animate-spin" />
|
||||||
|
{/if}
|
||||||
|
</h4>
|
||||||
|
|
||||||
|
{#if parsed_entries.length > 0}
|
||||||
|
<table class="table table-compact w-full text-xs">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Name</th>
|
||||||
|
<th>Date</th>
|
||||||
|
<th>Tags</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{#each parsed_entries as entry}
|
||||||
|
<tr>
|
||||||
|
<td class="truncate max-w-[200px]" title={entry.name}>{entry.name}</td>
|
||||||
|
<td>{entry.created_on?.substring(0,10)}</td>
|
||||||
|
<td>{entry.tags.join(', ')}</td>
|
||||||
|
</tr>
|
||||||
|
{/each}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{:else if files && files.length > 0 && !is_parsing}
|
||||||
|
<div class="text-center text-gray-500 py-4">No entries found in selected files.</div>
|
||||||
|
{:else if !files}
|
||||||
|
<div class="text-center text-gray-500 py-4">Select files to preview import.</div>
|
||||||
|
{/if}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Import Log -->
|
||||||
|
{#if import_log.length > 0}
|
||||||
|
<div class="bg-black text-green-400 p-2 rounded text-xs font-mono max-h-32 overflow-y-auto">
|
||||||
|
{#each import_log as log}
|
||||||
|
<div>{log}</div>
|
||||||
|
{/each}
|
||||||
|
</div>
|
||||||
|
{/if}
|
||||||
|
|
||||||
|
<div class="modal-action">
|
||||||
|
<button class="btn preset-tonal-secondary" onclick={onClose}>Cancel</button>
|
||||||
|
<button
|
||||||
|
class="btn preset-filled-primary"
|
||||||
|
disabled={parsed_entries.length === 0 || is_importing}
|
||||||
|
onclick={handle_import}
|
||||||
|
>
|
||||||
|
{#if is_importing}
|
||||||
|
Importing...
|
||||||
|
{:else}
|
||||||
|
Import {parsed_entries.length} Entries
|
||||||
|
{/if}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</Modal>
|
||||||
Reference in New Issue
Block a user