Add database

src/lib/collectors.js

@@ -3,6 +3,7 @@ const {request} = require("./utils/request")
 const {extractSharedData} = require("./utils/body")
 const {TtlCache, RequestCache} = require("./cache")
 const RequestHistory = require("./structures/RequestHistory")
+const db = require("./db")
 require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory)
 
 const requestCache = new RequestCache(constants.caching.resource_cache_time)
@@ -25,6 +26,10 @@ function fetchUser(username) {
 				const sharedData = extractSharedData(text)
 				const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
 				history.report("user", true)
+				if (constants.caching.db_user_id) {
+					db.prepare("INSERT OR IGNORE INTO Users (username, user_id) VALUES (@username, @user_id)")
+						.run({username: user.data.username, user_id: user.data.id})
+				}
 				return user
 			})
 		})
@@ -105,14 +110,18 @@ function fetchShortcodeData(shortcode) {
 				history.report("post", false)
 				console.error("missing data from post request, 429?", root) //todo: please make this better.
 				throw new Error("missing data from post request, 429?")
-			/** @type {import("./types").TimelineEntryN3} */
 			} else {
+				/** @type {import("./types").TimelineEntryN3} */
 				const data = root.data.shortcode_media
 				if (data == null) {
 					// the thing doesn't exist
 					throw constants.symbols.NOT_FOUND
 				} else {
 					history.report("post", true)
+					if (constants.caching.db_post_n3) {
+						db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)")
+							.run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)})
+					}
 					return data
 				}
 			}

src/lib/constants.js

@@ -16,7 +16,9 @@ let constants = {
 	caching: {
 		image_cache_control: `public, max-age=${7*24*60*60}`,
 		resource_cache_time: 30*60*1000,
-		instance_list_cache_time: 3*60*1000
+		instance_list_cache_time: 3*60*1000,
+		db_user_id: true,
+		db_post_n3: true
 	},
 
 	// Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream.
@@ -44,7 +46,9 @@ let constants = {
 		NOT_FOUND: Symbol("NOT_FOUND"),
 		NO_SHARED_DATA: Symbol("NO_SHARED_DATA"),
 		INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN")
-	}
+	},
+
+	database_version: 1
 }
 
 // Override values from config and export the result

src/lib/db.js

@@ -0,0 +1,10 @@
+const sqlite = require("better-sqlite3")
+const pj = require("path").join
+const fs = require("fs")
+
+const dir = pj(__dirname, "../../db")
+fs.mkdirSync(pj(dir, "backups"), {recursive: true})
+const db = new sqlite(pj(dir, "bibliogram.db"))
+module.exports = db
+
+require("./utils/upgradedb")()

src/lib/utils/upgradedb.js

@@ -0,0 +1,57 @@
+const constants = require("../constants")
+const pj = require("path").join
+const db = require("../db")
+require("../testimports")(db)
+
+const deltas = new Map([
+	// empty file to version 1
+	[1, function() {
+		db.prepare("DROP TABLE IF EXISTS Users")
+			.run()
+		db.prepare("DROP TABLE IF EXISTS DatabaseVersion")
+			.run()
+		db.prepare("DROP TABLE IF Exists Posts")
+			.run()
+
+		db.prepare("CREATE TABLE Users (username TEXT NOT NULL UNIQUE, user_id TEXT NOT NULL UNIQUE, PRIMARY KEY (username))")
+			.run()
+		db.prepare("CREATE TABLE DatabaseVersion (version INTEGER NOT NULL UNIQUE, PRIMARY KEY (version))")
+			.run()
+		db.prepare("CREATE TABLE Posts (shortcode TEXT NOT NULL UNIQUE, id TEXT NOT NULL UNIQUE, id_as_numeric NUMERIC NOT NULL, username TEXT NOT NULL, json TEXT NOT NULL, PRIMARY KEY (shortcode))")
+			// for future investigation: may not be able to sort by id as a string, may not be able to fit entire id in numeric type
+			.run()
+	}]
+])
+
+module.exports = async function() {
+	let currentVersion = 0
+	try {
+		currentVersion = db.prepare("SELECT version FROM DatabaseVersion").pluck().get() || 0
+	} catch (e) {}
+
+	const newVersion = constants.database_version
+
+	if (currentVersion !== newVersion) {
+		console.log(`Upgrading database from version ${currentVersion} to version ${newVersion}...`)
+		// go through the entire upgrade path
+		for (let entry = currentVersion+1; entry <= newVersion; entry++) {
+			// Back up current version
+			if (entry !== 1) {
+				const filename = `backups/bibliogram.db.bak-v${entry-1}`
+				process.stdout.write(`Backing up current to ${filename}... `)
+				await db.backup(pj(__dirname, "../../../db", filename))
+				process.stdout.write("done.\n")
+			}
+			// Run delta
+			process.stdout.write(`Using script ${entry}... `)
+			deltas.get(entry)()
+			db.prepare("DELETE FROM DatabaseVersion").run()
+			db.prepare("INSERT INTO DatabaseVersion (version) VALUES (?)").run(entry)
+			process.stdout.write("done.\n")
+		}
+		console.log(
+			   "Upgrade complete."
+			+"\n-----------------"
+		)
+	}
+}