Feat/Health check (#4047)

* Message on low log loss

* make console.log permanent

* Added: Health check option

* disable button

* change health check conditions

* i18n

* ./check

* Apply suggestions from code review

Co-authored-by: user1823 <92206575+user1823@users.noreply.github.com>

* delete shadowed fsrs

* Update ts/routes/deck-options/FsrsOptions.svelte

Co-authored-by: llama <gh@siid.sh>

* Update ftl/core/deck-config.ftl

Co-authored-by: user1823 <92206575+user1823@users.noreply.github.com>

* anon's suggestions

* snake_case

* capital slow

* make global

* on by default

* Adjusted loss values

* Show message on pass

* ./check

* ComputeParamsRequest

* update coefficients

* update thresholds

* fix thresholds

* Apply suggestions from code review

---------

Co-authored-by: user1823 <92206575+user1823@users.noreply.github.com>
Co-authored-by: llama <gh@siid.sh>
Co-authored-by: Damien Elmes <dae@users.noreply.github.com>

ftl/core/deck-config.ftl

@@ -395,6 +395,7 @@ deck-config-weights = FSRS parameters
 deck-config-compute-optimal-weights = Optimize FSRS parameters
 deck-config-compute-minimum-recommended-retention = Minimum recommended retention
 deck-config-optimize-button = Optimize Current Preset
+deck-config-health-check = Check health when optimizing (slow)
 deck-config-compute-button = Compute
 deck-config-ignore-before = Ignore cards reviewed before
 deck-config-time-to-optimize = It's been a while - using the Optimize All Presets button is recommended.
@@ -485,6 +486,15 @@ deck-config-percent-input = { $pct }%
 deck-config-optimizing-preset = Optimizing preset { $current_count }/{ $total_count }...
 deck-config-fsrs-must-be-enabled = FSRS must be enabled first.
 deck-config-fsrs-params-optimal = The FSRS parameters currently appear to be optimal.
+deck-config-fsrs-bad-fit-warning = Your memory is difficult for FSRS to predict. Recommendations:
+
+    - Suspend or reformulate leeches.
+    - Use the answer buttons consistently. Keep in mind that "Hard" is a passing grade, not a failing grade.
+    - Understand before you memorize.
+
+    If you follow these suggestions, performance will usually improve over the next few months.
+deck-config-fsrs-good-fit = FSRS is well adjusted to your memory.
+
 deck-config-fsrs-params-no-reviews = No reviews found. Make sure this preset is assigned to all decks (including subdecks) that you want to optimize, and try again.
 
 deck-config-wait-for-audio = Wait for audio

proto/anki/deck_config.proto

@@ -235,6 +235,7 @@ message DeckConfigsForUpdate {
   // only applies to v3 scheduler
   bool new_cards_ignore_review_limit = 7;
   bool fsrs = 8;
+  bool fsrs_health_check = 11;
   bool apply_all_parent_limits = 9;
   uint32 days_since_last_fsrs_optimize = 10;
 }
@@ -258,4 +259,5 @@ message UpdateDeckConfigsRequest {
   bool fsrs = 8;
   bool apply_all_parent_limits = 9;
   bool fsrs_reschedule = 10;
+  bool fsrs_health_check = 11;
 }

proto/anki/scheduler.proto

@@ -354,11 +354,13 @@ message ComputeFsrsParamsRequest {
   repeated float current_params = 2;
   int64 ignore_revlogs_before_ms = 3;
   uint32 num_of_relearning_steps = 4;
+  bool health_check = 5;
 }
 
 message ComputeFsrsParamsResponse {
   repeated float params = 1;
   uint32 fsrs_items = 2;
+  optional bool health_check_passed = 3;
 }
 
 message ComputeFsrsParamsFromItemsRequest {

rslib/src/config/bool.rs

@@ -40,6 +40,7 @@ pub enum BoolKey {
     WithScheduling,
     WithDeckConfigs,
     Fsrs,
+    FsrsHealthCheck,
     LoadBalancerEnabled,
     FsrsShortTermWithStepsEnabled,
     #[strum(to_string = "normalize_note_text")]
@@ -76,6 +77,7 @@ impl Collection {
             | BoolKey::RestorePositionBrowser
             | BoolKey::RestorePositionReviewer
             | BoolKey::LoadBalancerEnabled
+            | BoolKey::FsrsHealthCheck
             | BoolKey::NormalizeNoteText => self.get_config_optional(key).unwrap_or(true),
 
             // other options default to false

rslib/src/deckconfig/service.rs

@@ -192,6 +192,7 @@ impl From<anki_proto::deck_config::UpdateDeckConfigsRequest> for UpdateDeckConfi
             apply_all_parent_limits: c.apply_all_parent_limits,
             fsrs: c.fsrs,
             fsrs_reschedule: c.fsrs_reschedule,
+            fsrs_health_check: c.fsrs_health_check,
         }
     }
 }

rslib/src/deckconfig/update.rs

@@ -22,6 +22,7 @@ use crate::prelude::*;
 use crate::scheduler::fsrs::memory_state::UpdateMemoryStateEntry;
 use crate::scheduler::fsrs::memory_state::UpdateMemoryStateRequest;
 use crate::scheduler::fsrs::params::ignore_revlogs_before_ms_from_config;
+use crate::scheduler::fsrs::params::ComputeParamsRequest;
 use crate::search::JoinSearches;
 use crate::search::Negated;
 use crate::search::SearchNode;
@@ -41,6 +42,7 @@ pub struct UpdateDeckConfigsRequest {
     pub apply_all_parent_limits: bool,
     pub fsrs: bool,
     pub fsrs_reschedule: bool,
+    pub fsrs_health_check: bool,
 }
 
 impl Collection {
@@ -71,6 +73,7 @@ impl Collection {
             new_cards_ignore_review_limit: self.get_config_bool(BoolKey::NewCardsIgnoreReviewLimit),
             apply_all_parent_limits: self.get_config_bool(BoolKey::ApplyAllParentLimits),
             fsrs: self.get_config_bool(BoolKey::Fsrs),
+            fsrs_health_check: self.get_config_bool(BoolKey::FsrsHealthCheck),
             days_since_last_fsrs_optimize,
         })
     }
@@ -300,6 +303,7 @@ impl Collection {
             req.new_cards_ignore_review_limit,
         )?;
         self.set_config_bool_inner(BoolKey::ApplyAllParentLimits, req.apply_all_parent_limits)?;
+        self.set_config_bool_inner(BoolKey::FsrsHealthCheck, req.fsrs_health_check)?;
 
         Ok(())
     }
@@ -365,14 +369,15 @@ impl Collection {
             };
             let ignore_revlogs_before_ms = ignore_revlogs_before_ms_from_config(config)?;
             let num_of_relearning_steps = config.inner.relearn_steps.len();
-            match self.compute_params(
-                &search,
+            match self.compute_params(ComputeParamsRequest {
+                search: &search,
                 ignore_revlogs_before_ms,
-                idx as u32 + 1,
-                config_len,
-                config.fsrs_params(),
+                current_preset: idx as u32 + 1,
+                total_presets: config_len,
+                current_params: config.fsrs_params(),
                 num_of_relearning_steps,
-            ) {
+                health_check: false,
+            }) {
                 Ok(params) => {
                     println!("{}: {:?}", config.name, params.params);
                     config.inner.fsrs_params_6 = params.params;
@@ -452,6 +457,7 @@ mod test {
         col.set_config_string_inner(StringKey::CardStateCustomizer, "")?;
         col.set_config_bool_inner(BoolKey::NewCardsIgnoreReviewLimit, false)?;
         col.set_config_bool_inner(BoolKey::ApplyAllParentLimits, false)?;
+        col.set_config_bool_inner(BoolKey::FsrsHealthCheck, true)?;
 
         // pretend we're in sync
         let stamps = col.storage.get_collection_timestamps()?;
@@ -488,6 +494,7 @@ mod test {
             apply_all_parent_limits: false,
             fsrs: false,
             fsrs_reschedule: false,
+            fsrs_health_check: true,
         };
         assert!(!col.update_deck_configs(input.clone())?.changes.had_change());
 

rslib/src/scheduler/fsrs/params.rs

@@ -51,19 +51,46 @@ pub(crate) fn ignore_revlogs_before_ms_from_config(config: &DeckConfig) -> Resul
     ignore_revlogs_before_date_to_ms(&config.inner.ignore_revlogs_before_date)
 }
 
+pub struct ComputeParamsRequest<'t> {
+    pub search: &'t str,
+    pub ignore_revlogs_before_ms: TimestampMillis,
+    pub current_preset: u32,
+    pub total_presets: u32,
+    pub current_params: &'t Params,
+    pub num_of_relearning_steps: usize,
+    pub health_check: bool,
+}
+
+/// r: retention
+fn log_loss_adjustment(r: f32) -> f32 {
+    0.623 * (4. * r * (1. - r)).powf(0.738)
+}
+
+/// r: retention
+///
+/// c: review count
+fn rmse_adjustment(r: f32, c: u32) -> f32 {
+    0.0135 / (r.powf(0.504) - 1.14) + 0.176 / ((c as f32 / 1000.).powf(0.825) + 2.22) + 0.101
+}
+
 impl Collection {
     /// Note this does not return an error if there are less than 400 items -
     /// the caller should instead check the fsrs_items count in the return
     /// value.
     pub fn compute_params(
         &mut self,
-        search: &str,
-        ignore_revlogs_before: TimestampMillis,
-        current_preset: u32,
-        total_presets: u32,
-        current_params: &Params,
-        num_of_relearning_steps: usize,
+        request: ComputeParamsRequest,
     ) -> Result<ComputeFsrsParamsResponse> {
+        let ComputeParamsRequest {
+            search,
+            ignore_revlogs_before_ms: ignore_revlogs_before,
+            current_preset,
+            total_presets,
+            current_params,
+            num_of_relearning_steps,
+            health_check,
+        } = request;
+
         self.clear_progress();
         let timing = self.timing_today()?;
         let revlogs = self.revlog_for_srs(search)?;
@@ -75,6 +102,7 @@ impl Collection {
             return Ok(ComputeFsrsParamsResponse {
                 params: current_params.to_vec(),
                 fsrs_items,
+                health_check_passed: None,
             });
         }
         // adapt the progress handler to our built-in progress handling
@@ -108,12 +136,13 @@ impl Collection {
 
         let (progress, progress_thread) = create_progress_thread()?;
         let fsrs = FSRS::new(None)?;
-        let mut params = fsrs.compute_parameters(ComputeParametersInput {
+        let input = ComputeParametersInput {
             train_set: items.clone(),
             progress: Some(progress.clone()),
             enable_short_term: true,
             num_relearning_steps: Some(num_of_relearning_steps),
-        })?;
+        };
+        let mut params = fsrs.compute_parameters(input.clone())?;
         progress_thread.join().ok();
         if let Ok(current_fsrs) = FSRS::new(Some(current_params)) {
             let current_log_loss = current_fsrs.evaluate(items.clone(), |_| true)?.log_loss;
@@ -145,7 +174,34 @@ impl Collection {
             }
         }
 
-        Ok(ComputeFsrsParamsResponse { params, fsrs_items })
+        let health_check_passed = if health_check {
+            let fsrs = FSRS::new(None)?;
+            fsrs.evaluate_with_time_series_splits(input, |_| true)
+                .ok()
+                .map(|eval| {
+                    let r = items.iter().fold(0, |p, item| {
+                        p + (item
+                            .reviews
+                            .last()
+                            .map(|reviews| reviews.rating)
+                            .unwrap_or(0)
+                            > 1) as u32
+                    }) as f32
+                        / fsrs_items as f32;
+                    let adjusted_log_loss = eval.log_loss / log_loss_adjustment(r);
+                    let adjusted_rmse = eval.rmse_bins / rmse_adjustment(r, fsrs_items);
+
+                    adjusted_log_loss <= 1.11 || adjusted_rmse <= 1.53
+                })
+        } else {
+            None
+        };
+
+        Ok(ComputeFsrsParamsResponse {
+            params,
+            fsrs_items,
+            health_check_passed,
+        })
     }
 
     pub(crate) fn revlog_for_srs(

rslib/src/scheduler/service/mod.rs

@@ -23,6 +23,7 @@ use fsrs::FSRS;
 
 use crate::backend::Backend;
 use crate::prelude::*;
+use crate::scheduler::fsrs::params::ComputeParamsRequest;
 use crate::scheduler::new::NewCardDueOrder;
 use crate::scheduler::states::CardState;
 use crate::scheduler::states::SchedulingStates;
@@ -264,14 +265,15 @@ impl crate::services::SchedulerService for Collection {
         &mut self,
         input: scheduler::ComputeFsrsParamsRequest,
     ) -> Result<scheduler::ComputeFsrsParamsResponse> {
-        self.compute_params(
-            &input.search,
-            input.ignore_revlogs_before_ms.into(),
-            1,
-            1,
-            &input.current_params,
-            input.num_of_relearning_steps as usize,
-        )
+        self.compute_params(ComputeParamsRequest {
+            search: &input.search,
+            ignore_revlogs_before_ms: input.ignore_revlogs_before_ms.into(),
+            current_preset: 1,
+            total_presets: 1,
+            current_params: &input.current_params,
+            num_of_relearning_steps: input.num_of_relearning_steps as usize,
+            health_check: input.health_check,
+        })
     }
 
     fn simulate_fsrs_review(
@@ -372,7 +374,11 @@ impl crate::services::BackendSchedulerService for Backend {
             enable_short_term: true,
             num_relearning_steps: None,
         })?;
-        Ok(ComputeFsrsParamsResponse { params, fsrs_items })
+        Ok(ComputeFsrsParamsResponse {
+            params,
+            fsrs_items,
+            health_check_passed: None,
+        })
     }
 
     fn fsrs_benchmark(

ts/routes/deck-options/FsrsOptions.svelte

@@ -58,6 +58,8 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
     let computingParams = false;
     let checkingParams = false;
 
+    const healthCheck = state.fsrsHealthCheck;
+
     $: computing = computingParams || checkingParams;
     $: defaultparamSearch = `preset:"${state.getCurrentNameForSearch()}" -is:suspended`;
     $: roundedRetention = Number($config.desiredRetention.toFixed(2));
@@ -178,6 +180,7 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
                         ignoreRevlogsBeforeMs: getIgnoreRevlogsBeforeMs(),
                         currentParams: params,
                         numOfRelearningSteps: numOfRelearningStepsInDay,
+                        healthCheck: $healthCheck,
                     });
 
                     const already_optimal =
@@ -187,12 +190,22 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
                             )) ||
                         resp.params.length === 0;
 
-                    if (already_optimal) {
+                    if (resp.healthCheckPassed !== undefined) {
+                        if (resp.healthCheckPassed) {
+                            setTimeout(() => alert(tr.deckConfigFsrsGoodFit()), 200);
+                        } else {
+                            setTimeout(
+                                () => alert(tr.deckConfigFsrsBadFitWarning()),
+                                200,
+                            );
+                        }
+                    } else if (already_optimal) {
                         const msg = resp.fsrsItems
                             ? tr.deckConfigFsrsParamsOptimal()
                             : tr.deckConfigFsrsParamsNoReviews();
                         setTimeout(() => alert(msg), 200);
-                    } else {
+                    }
+                    if (!already_optimal) {
                         $config.fsrsParams6 = resp.params;
                         optimized = true;
                     }
@@ -312,6 +325,22 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
         placeholder={defaultparamSearch}
     />
 
+    <SwitchRow bind:value={$fsrsReschedule} defaultValue={false}>
+        <SettingTitle on:click={() => openHelpModal("rescheduleCardsOnChange")}>
+            <GlobalLabel title={tr.deckConfigRescheduleCardsOnChange()} />
+        </SettingTitle>
+    </SwitchRow>
+
+    {#if $fsrsReschedule}
+        <Warning warning={tr.deckConfigRescheduleCardsWarning()} />
+    {/if}
+
+    <SwitchRow bind:value={$healthCheck} defaultValue={false}>
+        <SettingTitle on:click={() => openHelpModal("deckConfigHealthCheck")}>
+            <GlobalLabel title={tr.deckConfigHealthCheck()} />
+        </SettingTitle>
+    </SwitchRow>
+
     <button
         class="btn {computingParams ? 'btn-warning' : 'btn-primary'}"
         disabled={!computingParams && computing}
@@ -323,17 +352,20 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
             {tr.deckConfigOptimizeButton()}
         {/if}
     </button>
-    <button
-        class="btn {checkingParams ? 'btn-warning' : 'btn-primary'}"
-        disabled={!checkingParams && computing}
-        on:click={() => checkParams()}
-    >
-        {#if checkingParams}
-            {tr.actionsCancel()}
-        {:else}
-            {tr.deckConfigEvaluateButton()}
-        {/if}
-    </button>
+    {#if false}
+        <!-- Can be re-enabled by some method in the future -->
+        <button
+            class="btn {checkingParams ? 'btn-warning' : 'btn-primary'}"
+            disabled={!checkingParams && computing}
+            on:click={() => checkParams()}
+        >
+            {#if checkingParams}
+                {tr.actionsCancel()}
+            {:else}
+                {tr.deckConfigEvaluateButton()}
+            {/if}
+        </button>
+    {/if}
     <div>
         {#if computingParams || checkingParams}
             {computeParamsProgressString}
@@ -351,18 +383,6 @@ License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
     </button>
 </div>
 
-<div class="m-2">
-    <SwitchRow bind:value={$fsrsReschedule} defaultValue={false}>
-        <SettingTitle on:click={() => openHelpModal("rescheduleCardsOnChange")}>
-            <GlobalLabel title={tr.deckConfigRescheduleCardsOnChange()} />
-        </SettingTitle>
-    </SwitchRow>
-
-    {#if $fsrsReschedule}
-        <Warning warning={tr.deckConfigRescheduleCardsWarning()} />
-    {/if}
-</div>
-
 <div class="m-2">
     <button class="btn btn-primary" on:click={() => (showSimulator = true)}>
         {tr.deckConfigFsrsSimulatorExperimental()}

ts/routes/deck-options/lib.ts

@@ -62,6 +62,7 @@ export class DeckOptionsState {
     readonly applyAllParentLimits: Writable<boolean>;
     readonly fsrs: Writable<boolean>;
     readonly fsrsReschedule: Writable<boolean> = writable(false);
+    readonly fsrsHealthCheck: Writable<boolean>;
     readonly daysSinceLastOptimization: Writable<number>;
     readonly currentPresetName: Writable<string>;
     /** Used to detect if there are any pending changes */
@@ -103,6 +104,7 @@ export class DeckOptionsState {
         this.newCardsIgnoreReviewLimit = writable(data.newCardsIgnoreReviewLimit);
         this.applyAllParentLimits = writable(data.applyAllParentLimits);
         this.fsrs = writable(data.fsrs);
+        this.fsrsHealthCheck = writable(data.fsrsHealthCheck);
         this.daysSinceLastOptimization = writable(data.daysSinceLastFsrsOptimize);
 
         // decrement the use count of the starting item, as we'll apply +1 to currently
@@ -267,6 +269,7 @@ export class DeckOptionsState {
             applyAllParentLimits: get(this.applyAllParentLimits),
             fsrs: get(this.fsrs),
             fsrsReschedule: get(this.fsrsReschedule),
+            fsrsHealthCheck: get(this.fsrsHealthCheck),
         };
     }