DataStore.ts 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387
  1. /**
  2. * @module lib/DataStore
  3. * This module contains the DataStore class, which is a general purpose, sync and async persistent JSON database - [see the documentation for more info](https://github.com/Sv443-Network/UserUtils/blob/main/docs.md#datastore)
  4. */
  5. import type { Prettify } from "./types.js";
  6. //#region types
  7. /** Function that takes the data in the old format and returns the data in the new format. Also supports an asynchronous migration. */
  8. type MigrationFunc = (oldData: any) => any | Promise<any>; // eslint-disable-line @typescript-eslint/no-explicit-any
  9. /** Dictionary of format version numbers and the function that migrates to them from the previous whole integer */
  10. export type DataMigrationsDict = Record<number, MigrationFunc>;
  11. /** Options for the DataStore instance */
  12. export type DataStoreOptions<TData> = Prettify<
  13. & {
  14. /**
  15. * A unique internal ID for this data store.
  16. * To avoid conflicts with other scripts, it is recommended to use a prefix that is unique to your script.
  17. * If you want to change the ID, you should make use of the {@linkcode DataStore.migrateId()} method.
  18. */
  19. id: string;
  20. /**
  21. * The default data object to use if no data is saved in persistent storage yet.
  22. * Until the data is loaded from persistent storage with {@linkcode DataStore.loadData()}, this will be the data returned by {@linkcode DataStore.getData()}.
  23. *
  24. * - ⚠️ This has to be an object that can be serialized to JSON using `JSON.stringify()`, so no functions or circular references are allowed, they will cause unexpected behavior.
  25. */
  26. defaultData: TData;
  27. /**
  28. * An incremental, whole integer version number of the current format of data.
  29. * If the format of the data is changed in any way, this number should be incremented, in which case all necessary functions of the migrations dictionary will be run consecutively.
  30. *
  31. * - ⚠️ Never decrement this number and optimally don't skip any numbers either!
  32. */
  33. formatVersion: number;
  34. /**
  35. * A dictionary of functions that can be used to migrate data from older versions to newer ones.
  36. * The keys of the dictionary should be the format version that the functions can migrate to, from the previous whole integer value.
  37. * The values should be functions that take the data in the old format and return the data in the new format.
  38. * The functions will be run in order from the oldest to the newest version.
  39. * If the current format version is not in the dictionary, no migrations will be run.
  40. */
  41. migrations?: DataMigrationsDict;
  42. /**
  43. * If an ID or multiple IDs are passed here, the data will be migrated from the old ID(s) to the current ID.
  44. * This will happen once per page load, when {@linkcode DataStore.loadData()} is called.
  45. * All future calls to {@linkcode DataStore.loadData()} in the session will not check for the old ID(s) anymore.
  46. * To migrate IDs manually, use the method {@linkcode DataStore.migrateId()} instead.
  47. */
  48. migrateIds?: string | string[];
  49. /**
  50. * Where the data should be saved (`"GM"` by default).
  51. * The protected methods {@linkcode DataStore.getValue()}, {@linkcode DataStore.setValue()} and {@linkcode DataStore.deleteValue()} are used to interact with the storage.
  52. * `"GM"` storage, `"localStorage"` and `"sessionStorage"` are supported out of the box, but in an extended class you can overwrite those methods to implement any other storage method.
  53. */
  54. storageMethod?: "GM" | "localStorage" | "sessionStorage";
  55. }
  56. & (
  57. // make sure that encodeData and decodeData are *both* either defined or undefined
  58. | {
  59. /**
  60. * Function to use to encode the data prior to saving it in persistent storage.
  61. * If this is specified, make sure to declare {@linkcode decodeData()} as well.
  62. *
  63. * You can make use of UserUtils' [`compress()`](https://github.com/Sv443-Network/UserUtils?tab=readme-ov-file#compress) function here to make the data use up less space at the cost of a little bit of performance.
  64. * @param data The input data as a serialized object (JSON string)
  65. */
  66. encodeData: (data: string) => string | Promise<string>,
  67. /**
  68. * Function to use to decode the data after reading it from persistent storage.
  69. * If this is specified, make sure to declare {@linkcode encodeData()} as well.
  70. *
  71. * You can make use of UserUtils' [`decompress()`](https://github.com/Sv443-Network/UserUtils?tab=readme-ov-file#decompress) function here to make the data use up less space at the cost of a little bit of performance.
  72. * @returns The resulting data as a valid serialized object (JSON string)
  73. */
  74. decodeData: (data: string) => string | Promise<string>,
  75. }
  76. | {
  77. encodeData?: never,
  78. decodeData?: never,
  79. }
  80. )
  81. >;
  82. //#region class
  83. /**
  84. * Manages a hybrid synchronous & asynchronous persistent JSON database that is cached in memory and persistently saved across sessions using [GM storage](https://wiki.greasespot.net/GM.setValue) (default), [localStorage](https://developer.mozilla.org/en-US/docs/Web/API/Window/localStorage) or [sessionStorage](https://developer.mozilla.org/en-US/docs/Web/API/Window/sessionStorage).
  85. * Supports migrating data from older format versions to newer ones and populating the cache with default data if no persistent data is found.
  86. * Can be overridden to implement any other storage method.
  87. *
  88. * All methods are `protected` or `public`, so you can easily extend this class and overwrite them to use a different storage method or to add other functionality.
  89. * Remember that you can use `super.methodName()` in the subclass to call the original method if needed.
  90. *
  91. * - ⚠️ The data is stored as a JSON string, so only data compatible with [`JSON.stringify()`](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/stringify) can be used. Circular structures and complex objects (containing functions, symbols, etc.) will either throw an error on load and save or cause otherwise unexpected behavior. Properties with a value of `undefined` will be removed from the data prior to saving it, so use `null` instead.
  92. * - ⚠️ If the storageMethod is left as the default of `"GM"`, the directives `@grant GM.getValue` and `@grant GM.setValue` are required. If you then also use the method {@linkcode DataStore.deleteData()}, the extra directive `@grant GM.deleteValue` is needed too.
  93. * - ⚠️ Make sure to call {@linkcode loadData()} at least once after creating an instance, or the returned data will be the same as `options.defaultData`
  94. *
  95. * @template TData The type of the data that is saved in persistent storage for the currently set format version (will be automatically inferred from `defaultData` if not provided)
  96. */
  97. export class DataStore<TData extends object = object> {
  98. public readonly id: string;
  99. public readonly formatVersion: number;
  100. public readonly defaultData: TData;
  101. public readonly encodeData: DataStoreOptions<TData>["encodeData"];
  102. public readonly decodeData: DataStoreOptions<TData>["decodeData"];
  103. public readonly storageMethod: Required<DataStoreOptions<TData>>["storageMethod"];
  104. private cachedData: TData;
  105. private migrations?: DataMigrationsDict;
  106. private migrateIds: string[] = [];
  107. /**
  108. * Creates an instance of DataStore to manage a sync & async database that is cached in memory and persistently saved across sessions.
  109. * Supports migrating data from older versions to newer ones and populating the cache with default data if no persistent data is found.
  110. *
  111. * - ⚠️ Requires the directives `@grant GM.getValue` and `@grant GM.setValue` if the storageMethod is left as the default of `"GM"`
  112. * - ⚠️ Make sure to call {@linkcode loadData()} at least once after creating an instance, or the returned data will be the same as `options.defaultData`
  113. *
  114. * @template TData The type of the data that is saved in persistent storage for the currently set format version (will be automatically inferred from `defaultData` if not provided) - **This has to be a JSON-compatible object!** (no undefined, circular references, etc.)
  115. * @param options The options for this DataStore instance
  116. */
  117. constructor(options: DataStoreOptions<TData>) {
  118. this.id = options.id;
  119. this.formatVersion = options.formatVersion;
  120. this.defaultData = options.defaultData;
  121. this.cachedData = options.defaultData;
  122. this.migrations = options.migrations;
  123. if(options.migrateIds)
  124. this.migrateIds = Array.isArray(options.migrateIds) ? options.migrateIds : [options.migrateIds];
  125. this.storageMethod = options.storageMethod ?? "GM";
  126. this.encodeData = options.encodeData;
  127. this.decodeData = options.decodeData;
  128. }
  129. //#region public
  130. /**
  131. * Loads the data saved in persistent storage into the in-memory cache and also returns it.
  132. * Automatically populates persistent storage with default data if it doesn't contain any data yet.
  133. * Also runs all necessary migration functions if the data format has changed since the last time the data was saved.
  134. */
  135. public async loadData(): Promise<TData> {
  136. try {
  137. if(this.migrateIds.length > 0) {
  138. await this.migrateId(this.migrateIds);
  139. this.migrateIds = [];
  140. }
  141. const gmData = await this.getValue(`_uucfg-${this.id}`, JSON.stringify(this.defaultData));
  142. let gmFmtVer = Number(await this.getValue(`_uucfgver-${this.id}`, NaN));
  143. if(typeof gmData !== "string") {
  144. await this.saveDefaultData();
  145. return { ...this.defaultData };
  146. }
  147. const isEncoded = Boolean(await this.getValue(`_uucfgenc-${this.id}`, false));
  148. let saveData = false;
  149. if(isNaN(gmFmtVer)) {
  150. await this.setValue(`_uucfgver-${this.id}`, gmFmtVer = this.formatVersion);
  151. saveData = true;
  152. }
  153. let parsed = await this.deserializeData(gmData, isEncoded);
  154. if(gmFmtVer < this.formatVersion && this.migrations)
  155. parsed = await this.runMigrations(parsed, gmFmtVer);
  156. if(saveData)
  157. await this.setData(parsed);
  158. this.cachedData = { ...parsed };
  159. return this.cachedData;
  160. }
  161. catch(err) {
  162. console.warn("Error while parsing JSON data, resetting it to the default value.", err);
  163. await this.saveDefaultData();
  164. return this.defaultData;
  165. }
  166. }
  167. /**
  168. * Returns a copy of the data from the in-memory cache.
  169. * Use {@linkcode loadData()} to get fresh data from persistent storage (usually not necessary since the cache should always exactly reflect persistent storage).
  170. * @param deepCopy Whether to return a deep copy of the data (default: `false`) - only necessary if your data object is nested and may have a bigger performance impact if enabled
  171. */
  172. public getData(deepCopy = false): TData {
  173. return deepCopy
  174. ? this.deepCopy(this.cachedData)
  175. : { ...this.cachedData };
  176. }
  177. /** Saves the data synchronously to the in-memory cache and asynchronously to the persistent storage */
  178. public setData(data: TData): Promise<void> {
  179. this.cachedData = data;
  180. const useEncoding = this.encodingEnabled();
  181. return new Promise<void>(async (resolve) => {
  182. await Promise.all([
  183. this.setValue(`_uucfg-${this.id}`, await this.serializeData(data, useEncoding)),
  184. this.setValue(`_uucfgver-${this.id}`, this.formatVersion),
  185. this.setValue(`_uucfgenc-${this.id}`, useEncoding),
  186. ]);
  187. resolve();
  188. });
  189. }
  190. /** Saves the default data passed in the constructor synchronously to the in-memory cache and asynchronously to persistent storage */
  191. public async saveDefaultData(): Promise<void> {
  192. this.cachedData = this.defaultData;
  193. const useEncoding = this.encodingEnabled();
  194. return new Promise<void>(async (resolve) => {
  195. await Promise.all([
  196. this.setValue(`_uucfg-${this.id}`, await this.serializeData(this.defaultData, useEncoding)),
  197. this.setValue(`_uucfgver-${this.id}`, this.formatVersion),
  198. this.setValue(`_uucfgenc-${this.id}`, useEncoding),
  199. ]);
  200. resolve();
  201. });
  202. }
  203. /**
  204. * Call this method to clear all persistently stored data associated with this DataStore instance.
  205. * The in-memory cache will be left untouched, so you may still access the data with {@linkcode getData()}
  206. * Calling {@linkcode loadData()} or {@linkcode setData()} after this method was called will recreate persistent storage with the cached or default data.
  207. *
  208. * - ⚠️ This requires the additional directive `@grant GM.deleteValue` if the storageMethod is left as the default of `"GM"`
  209. */
  210. public async deleteData(): Promise<void> {
  211. await Promise.all([
  212. this.deleteValue(`_uucfg-${this.id}`),
  213. this.deleteValue(`_uucfgver-${this.id}`),
  214. this.deleteValue(`_uucfgenc-${this.id}`),
  215. ]);
  216. }
  217. /** Returns whether encoding and decoding are enabled for this DataStore instance */
  218. public encodingEnabled(): this is Required<Pick<DataStoreOptions<TData>, "encodeData" | "decodeData">> {
  219. return Boolean(this.encodeData && this.decodeData);
  220. }
  221. //#region migrations
  222. /**
  223. * Runs all necessary migration functions consecutively and saves the result to the in-memory cache and persistent storage and also returns it.
  224. * This method is automatically called by {@linkcode loadData()} if the data format has changed since the last time the data was saved.
  225. * Though calling this method manually is not necessary, it can be useful if you want to run migrations for special occasions like a user importing potentially outdated data that has been previously exported.
  226. *
  227. * If one of the migrations fails, the data will be reset to the default value if `resetOnError` is set to `true` (default). Otherwise, an error will be thrown and no data will be saved.
  228. */
  229. public async runMigrations(oldData: unknown, oldFmtVer: number, resetOnError = true): Promise<TData> {
  230. if(!this.migrations)
  231. return oldData as TData;
  232. let newData = oldData;
  233. const sortedMigrations = Object.entries(this.migrations)
  234. .sort(([a], [b]) => Number(a) - Number(b));
  235. let lastFmtVer = oldFmtVer;
  236. for(const [fmtVer, migrationFunc] of sortedMigrations) {
  237. const ver = Number(fmtVer);
  238. if(oldFmtVer < this.formatVersion && oldFmtVer < ver) {
  239. try {
  240. const migRes = migrationFunc(newData);
  241. newData = migRes instanceof Promise ? await migRes : migRes;
  242. lastFmtVer = oldFmtVer = ver;
  243. }
  244. catch(err) {
  245. if(!resetOnError)
  246. throw new Error(`Error while running migration function for format version '${fmtVer}'`);
  247. console.error(`Error while running migration function for format version '${fmtVer}' - resetting to the default value.`, err);
  248. await this.saveDefaultData();
  249. return this.getData();
  250. }
  251. }
  252. }
  253. await Promise.all([
  254. this.setValue(`_uucfg-${this.id}`, await this.serializeData(newData as TData)),
  255. this.setValue(`_uucfgver-${this.id}`, lastFmtVer),
  256. this.setValue(`_uucfgenc-${this.id}`, this.encodingEnabled()),
  257. ]);
  258. return this.cachedData = { ...newData as TData };
  259. }
  260. /**
  261. * Tries to migrate the currently saved persistent data from one or more old IDs to the ID set in the constructor.
  262. * If no data exist for the old ID(s), nothing will be done, but some time may still pass trying to fetch the non-existent data.
  263. */
  264. public async migrateId(oldIds: string | string[]): Promise<void> {
  265. const ids = Array.isArray(oldIds) ? oldIds : [oldIds];
  266. await Promise.all(ids.map(async id => {
  267. const data = await this.getValue(`_uucfg-${id}`, JSON.stringify(this.defaultData));
  268. const fmtVer = Number(await this.getValue(`_uucfgver-${id}`, NaN));
  269. const isEncoded = Boolean(await this.getValue(`_uucfgenc-${id}`, false));
  270. if(data === undefined || isNaN(fmtVer))
  271. return;
  272. const parsed = await this.deserializeData(data, isEncoded);
  273. await Promise.allSettled([
  274. this.setValue(`_uucfg-${this.id}`, await this.serializeData(parsed)),
  275. this.setValue(`_uucfgver-${this.id}`, fmtVer),
  276. this.setValue(`_uucfgenc-${this.id}`, isEncoded),
  277. this.deleteValue(`_uucfg-${id}`),
  278. this.deleteValue(`_uucfgver-${id}`),
  279. this.deleteValue(`_uucfgenc-${id}`),
  280. ]);
  281. }));
  282. }
  283. //#region serialization
  284. /** Serializes the data using the optional this.encodeData() and returns it as a string */
  285. protected async serializeData(data: TData, useEncoding = true): Promise<string> {
  286. const stringData = JSON.stringify(data);
  287. if(!this.encodingEnabled() || !useEncoding)
  288. return stringData;
  289. const encRes = this.encodeData(stringData);
  290. if(encRes instanceof Promise)
  291. return await encRes;
  292. return encRes;
  293. }
  294. /** Deserializes the data using the optional this.decodeData() and returns it as a JSON object */
  295. protected async deserializeData(data: string, useEncoding = true): Promise<TData> {
  296. let decRes = this.encodingEnabled() && useEncoding ? this.decodeData(data) : undefined;
  297. if(decRes instanceof Promise)
  298. decRes = await decRes;
  299. return JSON.parse(decRes ?? data) as TData;
  300. }
  301. /** Copies a JSON-compatible object and loses all its internal references in the process */
  302. protected deepCopy<T>(obj: T): T {
  303. return JSON.parse(JSON.stringify(obj));
  304. }
  305. //#region storage
  306. /** Gets a value from persistent storage - can be overwritten in a subclass if you want to use something other than the default storage methods */
  307. protected async getValue<TValue extends GM.Value = string>(name: string, defaultValue: TValue): Promise<string | TValue> {
  308. switch(this.storageMethod) {
  309. case "localStorage":
  310. return localStorage.getItem(name) as TValue ?? defaultValue;
  311. case "sessionStorage":
  312. return sessionStorage.getItem(name) as string ?? defaultValue;
  313. default:
  314. return GM.getValue<TValue>(name, defaultValue);
  315. }
  316. }
  317. /**
  318. * Sets a value in persistent storage - can be overwritten in a subclass if you want to use something other than the default storage methods.
  319. * The default storage engines will stringify all passed values like numbers or booleans, so be aware of that.
  320. */
  321. protected async setValue(name: string, value: GM.Value): Promise<void> {
  322. switch(this.storageMethod) {
  323. case "localStorage":
  324. return localStorage.setItem(name, String(value));
  325. case "sessionStorage":
  326. return sessionStorage.setItem(name, String(value));
  327. default:
  328. return GM.setValue(name, String(value));
  329. }
  330. }
  331. /** Deletes a value from persistent storage - can be overwritten in a subclass if you want to use something other than the default storage methods */
  332. protected async deleteValue(name: string): Promise<void> {
  333. switch(this.storageMethod) {
  334. case "localStorage":
  335. return localStorage.removeItem(name);
  336. case "sessionStorage":
  337. return sessionStorage.removeItem(name);
  338. default:
  339. return GM.deleteValue(name);
  340. }
  341. }
  342. }