diff options
| -rw-r--r-- | setup.go | 77 | ||||
| -rw-r--r-- | ui.go | 32 |
2 files changed, 75 insertions, 34 deletions
@@ -37,7 +37,7 @@ func isDatabaseEmpty(db *sql.DB) tea.Cmd { } // Only populate the database if it is empty. - return isDBEmptyMsg(count == 0) + return isDictionaryEmptyMsg(count == 0) } } @@ -78,10 +78,29 @@ type SenseForDictionaryEntry struct { Example string } -func populateDictionary(rawDictionary string, db *sql.DB) tea.Cmd { +// dictionaryPopulator contains all the information required to populate the +// SQLite dictionary from the raw JSONL data. This is in a struct so that we can +// report progress back to the UI, then resume where we left off. +type dictionaryPopulator struct { + db *sql.DB + rawDictionaryPath string + langCode string + + tx *sql.Tx + stmt *sql.Stmt + tmpl *template.Template + scanner *bufio.Scanner + + totalLines int + currentLine int +} + +func setupPopulator(dp *dictionaryPopulator) tea.Cmd { return func() tea.Msg { + var err error + // Set up the template - tmpl, err := template.New("entry").Parse( + dp.tmpl, err = template.New("entry").Parse( `<p>{{ .Word }} {{ .Sound }} <i>{{ .POS }} {{ .Gender }}</i></p> <ol>{{ range .Senses}} <li class=sense>{{ .Sense }}<br> @@ -89,38 +108,44 @@ func populateDictionary(rawDictionary string, db *sql.DB) tea.Cmd { {{ end }}</ol> {{ if .Etymology }}<p><i>Étymologie: {{ .Etymology }}</i>{{ end }}`) if err != nil { - panic(err) + return errMsg(fmt.Errorf("preparing template: %w", err)) } - tx, err := db.Begin() + dp.tx, err = dp.db.Begin() if err != nil { return errMsg(fmt.Errorf("starting transaction: %w", err)) } // Set up a prepared statement - stmt, err := tx.Prepare("insert into words(word, definition) values(?, ?)") + dp.stmt, err = dp.tx.Prepare("insert into words(word, definition) values(?, ?)") if err != nil { return errMsg(fmt.Errorf("preparing statement: %w", err)) } - defer stmt.Close() - file, err := os.Open(rawDictionary) + file, err := os.Open(dp.rawDictionaryPath) if err != nil { return errMsg(fmt.Errorf("opening: %w", err)) } - defer file.Close() - var wordsAdded int - scanner := bufio.NewScanner(file) + dp.scanner = bufio.NewScanner(file) maxCapacity := 2_000_000 + buf := make([]byte, maxCapacity) - scanner.Buffer(buf, maxCapacity) + dp.scanner.Buffer(buf, maxCapacity) + + return populatingDictionaryMsg(dp) + } +} + +func populateDictionary(dp *dictionaryPopulator) tea.Cmd { + return func() tea.Msg { + for dp.scanner.Scan() { + dp.currentLine++ - for scanner.Scan() { var result rawDictionaryEntry - json.Unmarshal([]byte(scanner.Text()), &result) - if result.LangCode != "fr" { + json.Unmarshal([]byte(dp.scanner.Text()), &result) + if result.LangCode != dp.langCode { continue } @@ -171,33 +196,37 @@ func populateDictionary(rawDictionary string, db *sql.DB) tea.Cmd { } formattedDefinition := strings.Builder{} - err := tmpl.Execute(&formattedDefinition, entry) + err := dp.tmpl.Execute(&formattedDefinition, entry) if err != nil { return errMsg(fmt.Errorf("failed to render: %w", err)) } // Insert the entry - _, err = stmt.Exec(entry.Word, formattedDefinition.String()) + _, err = dp.stmt.Exec(entry.Word, formattedDefinition.String()) if err != nil { return errMsg(fmt.Errorf("inserting '%s': %w", entry.Word, err)) } - wordsAdded++ - + // Report status every once in a while by breaking out to the caller + if dp.currentLine%10000 == 0 { + return populatingDictionaryMsg(dp) + } } - if err := scanner.Err(); err != nil { + + // If we're outside of the loop, we either encountered an error, or it's + // time to commit the changes. + if err := dp.scanner.Err(); err != nil { return errMsg(fmt.Errorf("scanning: %w", err)) } - if err := tx.Commit(); err != nil { + if err := dp.tx.Commit(); err != nil { return errMsg(fmt.Errorf("committing: %w", err)) } - - _, err = db.Exec("create index wordindex on words(word);") + _, err := dp.db.Exec("create index wordindex on words(word);") if err != nil { return errMsg(fmt.Errorf("creating index: %s", err)) } - return isDBEmptyMsg(false) + return isDictionaryEmptyMsg(false) // We're done! } } @@ -19,6 +19,7 @@ type model struct { db *sql.DB c *http.Client p bluemonday.Policy + dp dictionaryPopulator wordInput textinput.Model definitionViewport viewport.Model @@ -37,10 +38,11 @@ type model struct { } type ( - errMsg error - definitionMsg string - wordAddedMsg string - isDBEmptyMsg bool + errMsg error + definitionMsg string + wordAddedMsg string + isDictionaryEmptyMsg bool + populatingDictionaryMsg *dictionaryPopulator ) func initialModel(c *http.Client, db *sql.DB, apiURL, ankiDeck, ankiModel, rawDictionary, firstWord string) model { @@ -66,10 +68,17 @@ func initialModel(c *http.Client, db *sql.DB, apiURL, ankiDeck, ankiModel, rawDi ), } + dp := dictionaryPopulator{ + db: db, + rawDictionaryPath: rawDictionary, + langCode: "fr", + } + return model{ db: db, c: c, p: *bluemonday.StrictPolicy(), + dp: dp, wordInput: input, definitionViewport: textbox, @@ -95,16 +104,19 @@ func (m model) Update(msg tea.Msg) (tea.Model, tea.Cmd) { var textPickerLongerCmds []tea.Cmd switch msg := msg.(type) { - case isDBEmptyMsg: - if bool(msg) { - // set up the database - return m, populateDictionary(m.rawDictionaryPath, m.db) - } else { + case isDictionaryEmptyMsg: + if bool(msg) { // We need to populate the dictionary + return m, setupPopulator(&m.dp) + } else { // The dictionary is ready m.dictionaryReady = true + m.statusString = "" if m.currentWord != "" { textPickerLongerCmds = append(textPickerLongerCmds, lookupWord(m.db, m.currentWord)) } } + case populatingDictionaryMsg: + m.statusString = fmt.Sprintf("Currently on line %d...", msg.currentLine) + return m, populateDictionary((*dictionaryPopulator)(msg)) case definitionMsg: m.currentDefinition = string(msg) m.err = nil @@ -170,7 +182,7 @@ func (m model) View() string { if m.err != nil { return fmt.Sprintf("Failed to load dictionary! Error:\n\n%s\n\nExit with Control-C.\n", m.err) } - return fmt.Sprintln("Preparing dictionary...") + return fmt.Sprintf("Preparing dictionary...\n\n%s\n\n", m.statusString) } func formatDefinitionForDisplay(policy bluemonday.Policy, definition string, maxWidth int) string { |