From 97cc884009dcec023aba614d62924da4f589e0e3 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 18:06:58 -0800 Subject: [PATCH 1/8] fix: Move workers to dist --- package-lock.json | 160 ++++++++++++++++++++++++++++++--------- package.json | 6 +- scripts/build-workers.js | 45 +++++++++++ 3 files changed, 175 insertions(+), 36 deletions(-) create mode 100644 scripts/build-workers.js diff --git a/package-lock.json b/package-lock.json index bf5341f..08d0a1d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "piparr", - "version": "0.0.1", + "version": "1.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "piparr", - "version": "0.0.1", + "version": "1.1.0", "license": "Zlib", "dependencies": { "@fastify/static": "^7.0.4", @@ -17,6 +17,7 @@ "moment": "^2.30.1", "node-ssdp": "^1.0.0", "react-bootstrap-typeahead": "^6.3.4", + "rimraf": "^6.0.1", "sqlite3": "^5.1.7" }, "devDependencies": { @@ -4975,6 +4976,58 @@ "node": ">=0.8.0" } }, + "node_modules/mv/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "optional": true, + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/mv/node_modules/glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", + "deprecated": "Glob versions prior to v9 are no longer supported", + "optional": true, + "dependencies": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mv/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "optional": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/mv/node_modules/rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", + "deprecated": "Rimraf versions prior to v4 are no longer supported", + "optional": true, + "dependencies": { + "glob": "^6.0.1" + }, + "bin": { + "rimraf": "bin.js" + } + }, "node_modules/napi-build-utils": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.2.tgz", @@ -5766,55 +5819,94 @@ "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==" }, "node_modules/rimraf": { - "version": "2.4.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", - "integrity": "sha512-J5xnxTyqaiw06JjMftq7L9ouA448dw/E7dKghkP9WpKNuwmARNNg+Gk8/u5ryb9N/Yo2+z3MCwuqFK/+qPOPfQ==", - "deprecated": "Rimraf versions prior to v4 are no longer supported", - "optional": true, + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-6.0.1.tgz", + "integrity": "sha512-9dkvaxAsk/xNXSJzMgFqqMCuFgt2+KsOFek3TMLfo8NCPfWpBmqwyNn5Y+NX56QUYfCtsyhF3ayiboEoUmJk/A==", "dependencies": { - "glob": "^6.0.1" + "glob": "^11.0.0", + "package-json-from-dist": "^1.0.0" }, "bin": { - "rimraf": "bin.js" + "rimraf": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/rimraf/node_modules/brace-expansion": { - "version": "1.1.11", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", - "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "optional": true, + "node_modules/rimraf/node_modules/glob": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.1.tgz", + "integrity": "sha512-zrQDm8XPnYEKawJScsnM0QzobJxlT/kHOOlRTio8IH/GrmxRE5fjllkzdaHclIuNjUQTJYH2xHNIGfdpJkDJUw==", "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" + "foreground-child": "^3.1.0", + "jackspeak": "^4.0.1", + "minimatch": "^10.0.0", + "minipass": "^7.1.2", + "package-json-from-dist": "^1.0.0", + "path-scurry": "^2.0.0" + }, + "bin": { + "glob": "dist/esm/bin.mjs" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/rimraf/node_modules/glob": { - "version": "6.0.4", - "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", - "integrity": "sha512-MKZeRNyYZAVVVG1oZeLaWie1uweH40m9AZwIwxyPbTSX4hHrVYSzLg0Ro5Z5R7XKkIX+Cc6oD1rqeDJnwsB8/A==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "optional": true, + "node_modules/rimraf/node_modules/jackspeak": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.2.tgz", + "integrity": "sha512-bZsjR/iRjl1Nk1UkjGpAzLNfQtzuijhn2g+pbZb98HQ1Gk8vM9hfbxeMBP+M2/UUdwj0RqGG3mlvk2MsAqwvEw==", "dependencies": { - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "2 || 3", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "@isaacs/cliui": "^8.0.2" }, "engines": { - "node": "*" + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/lru-cache": { + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.2.tgz", + "integrity": "sha512-123qHRfJBmo2jXDbo/a5YOQrJoHF/GNQTLzQ5+IdK5pWpceK17yRc6ozlWd25FxvGKQbIUs91fDFkXmDHTKcyA==", + "engines": { + "node": "20 || >=22" } }, "node_modules/rimraf/node_modules/minimatch": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", - "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", - "optional": true, + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", + "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" }, "engines": { - "node": "*" + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/rimraf/node_modules/path-scurry": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", + "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", + "dependencies": { + "lru-cache": "^11.0.0", + "minipass": "^7.1.2" + }, + "engines": { + "node": "20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" } }, "node_modules/safe-buffer": { diff --git a/package.json b/package.json index 8898eb0..41f66bb 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "moment": "^2.30.1", "node-ssdp": "^1.0.0", "react-bootstrap-typeahead": "^6.3.4", + "rimraf": "^6.0.1", "sqlite3": "^5.1.7" }, "devDependencies": { @@ -30,13 +31,14 @@ "webpack-cli": "^5.1.4" }, "name": "piparr", - "version": "1.0.0", + "version": "1.1.0", "description": "M3U/IPTV Proxy Server", "main": "dist/index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "build-server": "tsc server/src/index.ts --outDir ./dist --esModuleInterop", + "build-server": "tsc server/src/index.ts --outDir ./dist --esModuleInterop && npm run build-workers", "build-client": "webpack", + "build-workers": "node scripts/build-workers.js", "build": "npm run build-server && npm run build-client", "start": "node dist/index.js" }, diff --git a/scripts/build-workers.js b/scripts/build-workers.js new file mode 100644 index 0000000..ad9552d --- /dev/null +++ b/scripts/build-workers.js @@ -0,0 +1,45 @@ +const fs = require('fs'); +const path = require('path'); +const rimraf = require('rimraf'); + +// resolve directories +const serverSourceDirectory = path.resolve('./server/src'); +const serverSourceWorkersDirectory = path.resolve(path.join(serverSourceDirectory, 'workers')); + +const serverDistDirectory = path.resolve('./dist/') +const serverDistWorkersDirectory = path.resolve(path.join(serverDistDirectory, 'workers')); + +// remove workers directory if it exists, we need a fresh one +if (fs.existsSync(serverDistWorkersDirectory)) { + rimraf.rimrafSync(serverDistWorkersDirectory) +} + +// create the new directory +fs.mkdirSync(serverDistWorkersDirectory); + +// get all worker files in the source +const serverSourceWorkers = fs.readdirSync(serverSourceWorkersDirectory); + +// scan each worker file in the source +for(const sourceWorker of serverSourceWorkers) { + // get the directories for this file + const sourceFilePath = path.resolve(path.join(serverSourceWorkersDirectory, sourceWorker)); + const distFilePath = path.resolve(path.join(serverDistWorkersDirectory, sourceWorker)); + + // get the file type for this file + const sourceFileType = sourceWorker.split('.').pop(); + + if (sourceFileType.endsWith('ts')) { + // TODO: implement ts conversion for workers here + + console.error('[Build-Workers] TS conversion for workers is not supported yet, view ./scripts/build-workers for more info'); + + continue; + } + + if (sourceFileType.endsWith('js')) { + fs.copyFileSync(sourceFilePath, distFilePath); + } +} + +console.log(`[Build-Workers] built ${serverSourceWorkers.length} worker(s)`); \ No newline at end of file From 3e4cc679acf6e50257a0d8d4ca960707114136d7 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 18:18:39 -0800 Subject: [PATCH 2/8] feat: Streams can now be set to retry --- client/src/Streams.tsx | 37 ++++++++++++++++++++++++++++++++++--- server/src/StreamManager.ts | 2 +- server/src/WebServer.ts | 22 ++++++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/client/src/Streams.tsx b/client/src/Streams.tsx index 83bcc70..1e54416 100644 --- a/client/src/Streams.tsx +++ b/client/src/Streams.tsx @@ -8,7 +8,24 @@ export const StreamManager = () => { const providerReq = await fetch('/api/streams'); const providerRes = await providerReq.json(); - setStreams(providerRes); + setStreams(providerRes.map((i : any) => { + let providerState = 'Unknown State' + + if (i.healthy === 0) { + providerState = 'Failed' + } else if (i.healthy === 1) { + providerState = 'Healthy' + } else if (i.healthy === 2) { + providerState = 'Refreshing' + } else if (i.healthy === -1) { + providerState = 'Retrying' + } + + return { + ...i, + healthState: providerState + } + })); } const addProvider = async (event: React.FormEvent) => { @@ -45,6 +62,20 @@ export const StreamManager = () => { setStreams(streams.filter(i => i.id !== id)) } + const resetProvider = async(event : React.MouseEvent, id : string) => { + event.preventDefault(); + + const providerReq = await fetch('/api/streams/' + id + '/resetHealth', { + method: 'POST' + }); + + const providerRes = await providerReq.json(); + + setTimeout(() => { + fetchStreams(); + }, 500) + } + const confirmDeleteProvider = async (event : React.MouseEvent, id : string) => { if (confirm(`Are you sure you want to delete provider ${id}?`)) { await deleteProvider(event, id); @@ -81,8 +112,8 @@ export const StreamManager = () => { {provider.id} {provider.name} - {provider.healthy === 1 && Healthy}{provider.healthy === 0 && Failed}{provider.healthy === 2 && Refreshing} - confirmDeleteProvider(e, provider.id)}>Delete + {provider.healthState} + resetProvider(e, provider.id)}>Rescan confirmDeleteProvider(e, provider.id)}>Delete ); })} diff --git a/server/src/StreamManager.ts b/server/src/StreamManager.ts index 3321d92..fc56a39 100644 --- a/server/src/StreamManager.ts +++ b/server/src/StreamManager.ts @@ -22,7 +22,7 @@ export default class StreamManager { console.log(`[Piparr][StreamManager] fetching streams`); // select all streams from database - const streams = await DatabaseEngine.All('SELECT * FROM streams WHERE healthy = 1;') as Stream[]; + const streams = await DatabaseEngine.All('SELECT * FROM streams WHERE healthy = 1 OR healthy = -1;') as Stream[]; // run operation with all streams for(const stream of streams) { diff --git a/server/src/WebServer.ts b/server/src/WebServer.ts index 3d1c8a1..e66acb2 100644 --- a/server/src/WebServer.ts +++ b/server/src/WebServer.ts @@ -74,6 +74,28 @@ export default class WebServer { res.send(true); }); + fastify.post('/api/streams/:streamId/resetHealth', async (req, res) => { + const params = req.params as any; + + console.log(params) + + const streams = await DatabaseEngine.AllSafe(`SELECT * FROM streams WHERE id = ?;`, [params.streamId]) as Stream[]; + + if (streams.length === 0) { + console.warn(`the requested stream was not found`) + + res.status(404); + + res.send(404); + + return; + } + + await DatabaseEngine.RunSafe(`UPDATE streams SET healthy = -1 WHERE id = ?`, [ params.streamId ]); + + res.send(true); + }); + fastify.get('/api/streams/:streamId/sources', async (req, res) => { const params = req.params as any; From 675adb5ed7c6b501ee3e4c9715922c70b16a25a5 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 18:19:53 -0800 Subject: [PATCH 3/8] fix: Remove unneeded console.log --- client/src/Channel.tsx | 4 ---- server/src/WebServer.ts | 8 -------- 2 files changed, 12 deletions(-) diff --git a/client/src/Channel.tsx b/client/src/Channel.tsx index 80ab2d5..6697941 100644 --- a/client/src/Channel.tsx +++ b/client/src/Channel.tsx @@ -47,8 +47,6 @@ export const ChannelManager = () => { const sourcesReq = await fetch('/api/channels/' + (params as any).channelId + '/streams'); const sourcesRes = await sourcesReq.json(); - console.log(sourcesRes) - setStreamSources(sourcesRes.sources); setSelectedSources(sourcesRes.sources); setSelectedStreams(sourcesRes.streams.map((i : any) => { @@ -67,8 +65,6 @@ export const ChannelManager = () => { const fetchStreamSourcesForSelected = async (selected: any[]) => { setSelectedStreams(selected); - console.log(selected) - // remove any sources from deselected streams const selectedSourcesUpdated = selectedSources.filter(i => selected.filter(ii => ii.id === i.stream).length > 0); diff --git a/server/src/WebServer.ts b/server/src/WebServer.ts index e66acb2..9edf7ba 100644 --- a/server/src/WebServer.ts +++ b/server/src/WebServer.ts @@ -42,8 +42,6 @@ export default class WebServer { fastify.post('/api/streams', async (req, res) => { const payload = req.body as any; - console.log(payload) - const streamId = await DatabaseEngine.Insert('INSERT INTO streams (name, stream, connections, last_updated, type, regex) VALUES (?, ?, ?, ?, ?, ?);', [ payload.name, payload.stream, @@ -67,8 +65,6 @@ export default class WebServer { fastify.delete('/api/streams/:streamId', async (req, res) => { const params = req.params as any; - console.log(params) - await DatabaseEngine.RunSafe(`DELETE FROM streams WHERE id = ?;`, [params.streamId]); res.send(true); @@ -77,8 +73,6 @@ export default class WebServer { fastify.post('/api/streams/:streamId/resetHealth', async (req, res) => { const params = req.params as any; - console.log(params) - const streams = await DatabaseEngine.AllSafe(`SELECT * FROM streams WHERE id = ?;`, [params.streamId]) as Stream[]; if (streams.length === 0) { @@ -131,8 +125,6 @@ export default class WebServer { fastify.post('/api/channels', async (req, res) => { const payload = req.body as any; - console.log(payload) - const streamId = await DatabaseEngine.Insert(`INSERT INTO channels (name, logo, epg, channel_number) VALUES (?, ?, ?, ?);`, [ payload.name, payload.logo, From 17785787580b2fe046161c87dc4249b631e02928 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 20:01:00 -0800 Subject: [PATCH 4/8] chore: Comment server source --- server/src/Advertise.ts | 3 ++ server/src/BackgroundThreading.ts | 22 ++++++++++++++- server/src/DatabaseEngine.ts | 47 +++++++++++++++++++++++++++++++ server/src/Timers.ts | 3 ++ server/src/WebServer.ts | 46 ++++++++++++++++++++++++++++-- server/src/index.ts | 11 ++++++++ 6 files changed, 129 insertions(+), 3 deletions(-) diff --git a/server/src/Advertise.ts b/server/src/Advertise.ts index 5b75c4e..d773620 100644 --- a/server/src/Advertise.ts +++ b/server/src/Advertise.ts @@ -1,5 +1,8 @@ import { Server as SSDP } from 'node-ssdp'; +/** + * SSDP Advertising Service + */ export class Advertise { private server: SSDP; diff --git a/server/src/BackgroundThreading.ts b/server/src/BackgroundThreading.ts index 67406e3..9518bd2 100644 --- a/server/src/BackgroundThreading.ts +++ b/server/src/BackgroundThreading.ts @@ -1,12 +1,28 @@ import { argv } from "node:process"; import { Worker } from "node:worker_threads"; +/** + * BackgroundThreading + * + * Allows large blocking tasks to be ran on the background using Workers. + * + */ export abstract class BackgroundThreading { + /** + * Run a defined worker file in the background. + * @param workerFile The path for the file to run. + * @param payload The payload to pass into the worker. + * @param callback The callback to be run once the worker emits a message. + * @param timeout Should the worker be terminated after a time, useful for things that might hang. + */ public static Run(workerFile: string, payload : any, callback : (err : Error | null, data: any) => void, timeout : number = 0) { + // track if we fired the callback let callbackCalled = false; + // create the worker const worker = new Worker(workerFile, { workerData: payload }); + // listen for worker message worker.on('message', (data) => { if (!callbackCalled) { callbackCalled = true; @@ -14,7 +30,8 @@ export abstract class BackgroundThreading { callback(null, data); } }); - + + // if worker errors out, terminate it pass error to callback worker.on('error', (error) => { console.log(`[Piparr][BackgroundThreading] background task has encountered an error`, error); @@ -27,15 +44,18 @@ export abstract class BackgroundThreading { } }); + // debug for when worker exits worker.on('exit', () => { console.log(`[Piparr][BackgroundThreading] background task has stopped`); }) + // timeout only if timeout var is provided if (timeout > 0) { setTimeout(() => { if (!callbackCalled) { callbackCalled = true; + // terminate worker if timeout reached worker.terminate(); callback(new Error(`Background task failed, thread surpassed timeout of ${timeout}ms`), null); diff --git a/server/src/DatabaseEngine.ts b/server/src/DatabaseEngine.ts index 765e612..0fe91d0 100644 --- a/server/src/DatabaseEngine.ts +++ b/server/src/DatabaseEngine.ts @@ -2,9 +2,18 @@ import path from 'node:path'; import sqlite3 from "sqlite3"; +/** + * DatabaseEngine + * + * Easy interface for SQLite3. + */ export abstract class DatabaseEngine { public static instance: sqlite3.Database; + /** + * Init ensures the database exists and contains tables and basic settings + * @returns void + */ public static Init() { return new Promise((resolve, reject) => { const databasePath = path.resolve('./data/app.db'); @@ -33,6 +42,12 @@ export abstract class DatabaseEngine { }) } + /** + * Run a SQL query + * @param sql + * @returns + * @deprecated Use RunSafe instead to prevent against SQL injection. + */ public static Run(sql: string) { return new Promise((resolve, reject) => { console.log(`[Piparr][database][async][run] running ${sql}`) @@ -49,6 +64,11 @@ export abstract class DatabaseEngine { }) } + /** + * Run a SQL query with variable safety + * @param sql + * @returns + */ public static RunSafe(sql: string, object: any) { return new Promise((resolve, reject) => { console.log(`[Piparr][database][async][run] running ${sql}`) @@ -65,6 +85,12 @@ export abstract class DatabaseEngine { }) } + /** + * Run a SQL insert with variable safety + * @param sql + * @param object + * @returns + */ public static Insert(sql: string, object: any) { return new Promise((resolve, reject) => { console.log(`[Piparr][database][async][run] running ${sql}`) @@ -81,6 +107,11 @@ export abstract class DatabaseEngine { }) } + /** + * Run a SQL update + * @param sql + * @returns + */ public static Update(sql: string, object: any) { return new Promise((resolve, reject) => { console.log(`[Piparr][database][async][run] running ${sql}`) @@ -97,6 +128,13 @@ export abstract class DatabaseEngine { }) } + /** + * Run an SQL query and return all + * @param sql + * @param object + * @returns + * @deprecated Use AllSafe to prevent against sql injection + */ public static All(sql: string) { return new Promise((resolve, reject) => { console.log(`[Piparr][database][async][all] running ${sql}`) @@ -113,6 +151,12 @@ export abstract class DatabaseEngine { }) } + /** + * Run an SQL query and return all + * @param sql + * @param object + * @returns + */ public static AllSafe(sql: string, object: any) { return new Promise((resolve, reject) => { console.log(`[Piparr][database][async][all] running ${sql}`) @@ -129,6 +173,9 @@ export abstract class DatabaseEngine { }) } + /** + * Create default tables when database is initialized + */ private static async CreateTables() { await this.Run(`CREATE TABLE IF NOT EXISTS settings ( key TEXT PRIMARY KEY, diff --git a/server/src/Timers.ts b/server/src/Timers.ts index 47ae407..d9add41 100644 --- a/server/src/Timers.ts +++ b/server/src/Timers.ts @@ -1,3 +1,6 @@ +/** + * Timers provides easy promise based timers for async + */ export abstract class Timers { public static WaitFor(time: number) { return new Promise((resolve, reject) => { diff --git a/server/src/WebServer.ts b/server/src/WebServer.ts index 9edf7ba..da93e89 100644 --- a/server/src/WebServer.ts +++ b/server/src/WebServer.ts @@ -10,21 +10,32 @@ import StreamManager from "./StreamManager"; import { DatabaseEngine } from './DatabaseEngine'; import { Channel, ChannelSource, EpgBuilder, Stream } from './types'; +/** + * WebServer provides the routes and general setup for Piparr + */ export default class WebServer { + /** + * Start our web server + */ public static Run() { + // Create the fastify web server with logger enabled const fastify = Fastify({ logger: true }); + //#region Static Routes + // Register our static routes fastify.register(FastifyStatic, { root: path.resolve('./static'), prefix: '/static/', }); + // Index route redirect for /web/index.html fastify.get('/', (req, res) => { res.redirect('/web/index.html', 302); }); + // Main web app fastify.get('/web/index.html', (req, res) => { const viewsPath = path.resolve(path.join('./views', 'web.html')); const webAppFile = fs.readFileSync(viewsPath).toString(); @@ -32,13 +43,17 @@ export default class WebServer { res.header('content-type', 'text/html'); res.send(webAppFile); }); + //#endregion + //#region API Routes + // API route to return streams fastify.get('/api/streams', async (req, res) => { const streams = await DatabaseEngine.All(`SELECT * FROM streams;`); res.send(streams); }); + // API route to create streams fastify.post('/api/streams', async (req, res) => { const payload = req.body as any; @@ -62,6 +77,7 @@ export default class WebServer { }); }); + // API route to delete streams fastify.delete('/api/streams/:streamId', async (req, res) => { const params = req.params as any; @@ -70,6 +86,7 @@ export default class WebServer { res.send(true); }); + // API route to reset health on streams fastify.post('/api/streams/:streamId/resetHealth', async (req, res) => { const params = req.params as any; @@ -90,6 +107,7 @@ export default class WebServer { res.send(true); }); + // API route to get sources for streams fastify.get('/api/streams/:streamId/sources', async (req, res) => { const params = req.params as any; @@ -116,12 +134,14 @@ export default class WebServer { }))); }); + // API route to get channels created fastify.get('/api/channels', async (req, res) => { const channels = await DatabaseEngine.All(`SELECT * FROM channels;`); res.send(channels); }); + // API route to create a new channel fastify.post('/api/channels', async (req, res) => { const payload = req.body as any; @@ -141,6 +161,7 @@ export default class WebServer { }); }); + // API route to get information about a channel fastify.get('/api/channels/:channelId', async (req, res) => { const params = req.params as any; @@ -166,6 +187,7 @@ export default class WebServer { }) }); + // API route to update stream sources connected to a channel fastify.put('/api/channels/:channelId/streams', async (req, res) => { const params = req.params as any; const body = req.body as any; @@ -184,13 +206,17 @@ export default class WebServer { const channel = channels[0]; + // fetch all streams const streams = await DatabaseEngine.All(`SELECT * FROM streams`) as Stream[]; + + // fetch all current sources for this channel const sources = await DatabaseEngine.AllSafe(`SELECT * FROM channel_source WHERE channel_id = ?`, [ channel.id ]) as ChannelSource[]; // loop through new provided streams for(const sourceId of body.sources) { const existingSource = sources.find(i => i.stream_channel === sourceId); + // if channel already exists we skip if (typeof existingSource !== 'undefined') { console.log(`[Piparr] Skipping add of source ${sourceId}`); @@ -226,7 +252,7 @@ export default class WebServer { ]); } - // remove invalidated sources + // remove invalidated sources, channels that exist in the db and not in the new payload will be removed for(const existingSource of sources) { if (body.sources.includes(existingSource)) { console.log(`[Piparr] source ${existingSource.stream_id} from stream ${existingSource.stream_channel} validated`); @@ -242,6 +268,7 @@ export default class WebServer { res.send(200) }); + // delete all source streams from a channel fastify.delete('/api/channels/:channelId/streams', async (req, res) => { const params = req.params as any; const body = req.body as any; @@ -265,6 +292,7 @@ export default class WebServer { res.send(200) }); + // get the source streams for a channel fastify.get('/api/channels/:channelId/streams', async (req, res) => { const params = req.params as any; @@ -282,12 +310,16 @@ export default class WebServer { const channel = channels[0]; + // fetch all streams + const streams = await DatabaseEngine.All(`SELECT * FROM streams`) as Stream[]; + + // fetch all current sources for this channel const sources = await DatabaseEngine.AllSafe(`SELECT * FROM channel_source WHERE channel_id = ?`, [ channel.id ]) as ChannelSource[]; - const streams = await DatabaseEngine.All(`SELECT * FROM streams;`) as Stream[]; let selectedStreams : any[] = []; let selectedSources: any[] = []; + // we need to do some pairing up here to display correctly for(const source of sources) { const sourceStream = StreamManager.streams.find(i => i.id === source.stream_channel); @@ -314,7 +346,10 @@ export default class WebServer { sources: selectedSources }) }); + //#endregion + //#region HDHomeRun Routes + // HDHomeRun Route to get raw video source of stream fastify.get('/channels/:number/video', async (req, res) => { const params = req.params as any; @@ -363,6 +398,7 @@ export default class WebServer { res.redirect(sourceStream.endpoint, 302); }); + // Generate the EPG data into an XMLTV output fastify.get('/guide.xml', async (req, res) => { console.log(`[Piparr] Request to build guide`); @@ -418,6 +454,7 @@ export default class WebServer { res.send(epgOut); }); + // Get information about the HDHomeRun device fastify.get('/device.xml', (req, res) => { const host = req.protocol + '://' + req.hostname; return res @@ -425,6 +462,7 @@ export default class WebServer { .send(Advertise.Instance().getHdhrDeviceXml(host)); }); + // Get information about the HDHomeRun device fastify.get('/discover.json', (req, res) => { return res.send( Advertise.Instance().getHdhrDevice( @@ -433,6 +471,7 @@ export default class WebServer { ); }); + // Get information about the HDHomeRun device, specifically our capabilities fastify.get('/lineup_status.json', (req, res) => { return res.send({ ScanInProgress: 0, @@ -442,6 +481,7 @@ export default class WebServer { }); }); + // Display the lineup we have enabled to emulate an HDHomeRUn device fastify.get('/lineup.json', async (req, res) => { const storedChannels = await DatabaseEngine.All(`SELECT * FROM channels;`) as Channel[]; @@ -457,7 +497,9 @@ export default class WebServer { return res.send(lineup); }); + //#endregion + // Make the server listen to accept traffic fastify.listen({ host: '0.0.0.0', port: 34400 }, (err, addr) => { if (err) throw err; diff --git a/server/src/index.ts b/server/src/index.ts index 3f92042..00a5384 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -6,9 +6,13 @@ import WebServer from "./WebServer"; import fs from 'node:fs'; import path from 'node:path'; +/** + * Main application for Piparr + */ export default class Piparr { public static dataDir: string; + // make sure data directory exists public static async CreateDirectories() { const dataPath = path.resolve('./data'); @@ -19,19 +23,26 @@ export default class Piparr { this.dataDir = dataPath; } + // main init function public static async main() : Promise { + // create directories this.CreateDirectories() + // create database library await DatabaseEngine.Init(); + // advertise our HDHomeRun Device Advertise.Instance(); + // begin monitoring streams StreamManager.MonitorStreams(); + // run the web server WebServer.Run(); } } +// start our application Piparr.main().catch(e => { console.error(e); From acb7c76f9eb0df47724bd45586dcdbd00380d8b0 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 20:10:48 -0800 Subject: [PATCH 5/8] chore: Comment client source --- client/src/Channel.tsx | 9 ++++++++- client/src/Channels.tsx | 4 ++++ client/src/Root.tsx | 2 +- client/src/Streams.tsx | 6 ++++++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/client/src/Channel.tsx b/client/src/Channel.tsx index 6697941..8bae7df 100644 --- a/client/src/Channel.tsx +++ b/client/src/Channel.tsx @@ -19,6 +19,7 @@ export const ChannelManager = () => { const params = useParams(); + // fetch channel info const fetchChannel = async () => { const channelReq = await fetch('/api/channels/' + (params as any).channelId); const channelRes = await channelReq.json(); @@ -26,10 +27,12 @@ export const ChannelManager = () => { setChannel(channelRes); } + // fetch stream info for all of streams const fetchStreams = async () => { const channelReq = await fetch('/api/streams'); const channelRes = await channelReq.json(); + // we need to do a map to show stream health setStreams(channelRes.map((i : any) => { let status = 'Healthy'; @@ -43,6 +46,7 @@ export const ChannelManager = () => { })); } + // fetch all sources for streams const fetchStreamSources = async () => { const sourcesReq = await fetch('/api/channels/' + (params as any).channelId + '/streams'); const sourcesRes = await sourcesReq.json(); @@ -62,6 +66,7 @@ export const ChannelManager = () => { })); } + // get sources for streams that we wish to use for our channel const fetchStreamSourcesForSelected = async (selected: any[]) => { setSelectedStreams(selected); @@ -83,6 +88,7 @@ export const ChannelManager = () => { setStreamSources(sources); } + // send a PUT request to update the stream sources for this channel const updateChannelSources = async (event: React.FormEvent) => { event.preventDefault(); @@ -97,6 +103,7 @@ export const ChannelManager = () => { const channelRes : any = await channelReq.json(); } + // Delete the channel stream sources, clean wipe const deleteChannelSources = async (event: React.FormEvent) => { event.preventDefault(); @@ -110,7 +117,7 @@ export const ChannelManager = () => { setSelectedStreams([]); } - + // Get the basic channel info const fetchChannelData = async () => { await fetchStreams() diff --git a/client/src/Channels.tsx b/client/src/Channels.tsx index 1146fbc..509413a 100644 --- a/client/src/Channels.tsx +++ b/client/src/Channels.tsx @@ -5,6 +5,7 @@ import { FormDataCommon } from './common/FormDataCommon'; export const ChannelsLineup = () => { const [channels, setChannels] = React.useState([]); + // Get all channels const fetchChannels = async () => { const channelReq = await fetch('/api/channels'); const channelRes = await channelReq.json(); @@ -12,6 +13,7 @@ export const ChannelsLineup = () => { setChannels(channelRes); } + // Create a channel const addChannel = async (event: React.FormEvent) => { event.preventDefault(); @@ -34,6 +36,7 @@ export const ChannelsLineup = () => { setChannels(([] as any).concat(channels, [ channelRes ])) } + // Request a channel be deleted const deleteChannel = async(event : React.MouseEvent, id : string) => { event.preventDefault(); @@ -46,6 +49,7 @@ export const ChannelsLineup = () => { setChannels(channels.filter(i => i.id !== id)) } + // Confirm we want to delete the channel const confirmDeleteChannel = async (event : React.MouseEvent, id : string) => { if (confirm(`Are you sure you want to delete channel id ${id}?`)) { await deleteChannel(event, id); diff --git a/client/src/Root.tsx b/client/src/Root.tsx index f6a83b5..3ba77d2 100644 --- a/client/src/Root.tsx +++ b/client/src/Root.tsx @@ -4,7 +4,7 @@ import { ChannelsLineup } from './Channels'; import { StreamManager } from './Streams'; import { ChannelManager } from './Channel'; -// mount app on dom +// mount app on dom, provide basic web app frame export const AppRoot = () => { return(
diff --git a/client/src/Streams.tsx b/client/src/Streams.tsx index 1e54416..83d8115 100644 --- a/client/src/Streams.tsx +++ b/client/src/Streams.tsx @@ -4,6 +4,7 @@ import { FormDataCommon } from './common/FormDataCommon'; export const StreamManager = () => { const [streams, setStreams] = React.useState([]); + // fetch all streams, add health to display on label const fetchStreams = async () => { const providerReq = await fetch('/api/streams'); const providerRes = await providerReq.json(); @@ -28,6 +29,8 @@ export const StreamManager = () => { })); } + // TODO: rename provider to streams + // add a new provider (stream) const addProvider = async (event: React.FormEvent) => { event.preventDefault(); @@ -50,6 +53,7 @@ export const StreamManager = () => { setStreams(([] as any).concat(streams, [ providerRes ])) } + // delete a provider (stream) const deleteProvider = async(event : React.MouseEvent, id : string) => { event.preventDefault(); @@ -62,6 +66,7 @@ export const StreamManager = () => { setStreams(streams.filter(i => i.id !== id)) } + // reset health on a provider const resetProvider = async(event : React.MouseEvent, id : string) => { event.preventDefault(); @@ -76,6 +81,7 @@ export const StreamManager = () => { }, 500) } + // Confirm that you want to delete a stream const confirmDeleteProvider = async (event : React.MouseEvent, id : string) => { if (confirm(`Are you sure you want to delete provider ${id}?`)) { await deleteProvider(event, id); From 7ae75b5b43080d5b956091057a86881e75af2006 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 20:14:48 -0800 Subject: [PATCH 6/8] fix: Delete channels --- client/src/Channels.tsx | 2 +- server/src/WebServer.ts | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/client/src/Channels.tsx b/client/src/Channels.tsx index 509413a..784e650 100644 --- a/client/src/Channels.tsx +++ b/client/src/Channels.tsx @@ -40,7 +40,7 @@ export const ChannelsLineup = () => { const deleteChannel = async(event : React.MouseEvent, id : string) => { event.preventDefault(); - const channelReq = await fetch('/api/streams/' + id, { + const channelReq = await fetch('/api/channels/' + id, { method: 'DELETE' }); diff --git a/server/src/WebServer.ts b/server/src/WebServer.ts index da93e89..7d07ea0 100644 --- a/server/src/WebServer.ts +++ b/server/src/WebServer.ts @@ -268,6 +268,32 @@ export default class WebServer { res.send(200) }); + // delete all source streams from a channel + fastify.delete('/api/channels/:channelId', async (req, res) => { + const params = req.params as any; + const body = req.body as any; + + const channels = await DatabaseEngine.AllSafe('SELECT * FROM channels WHERE id = ?;', [params.channelId]) as Channel[]; + + if (channels.length === 0) { + console.warn(`the requested channel was not found`) + + res.status(404); + + res.send(404); + + return; + } + + const channel = channels[0]; + + await DatabaseEngine.AllSafe(`DELETE FROM channels WHERE id = ?`, [ channel.id ]) as ChannelSource[]; + + await DatabaseEngine.AllSafe(`DELETE FROM channel_source WHERE channel_id = ?`, [ channel.id ]) as ChannelSource[]; + + res.send(200) + }); + // delete all source streams from a channel fastify.delete('/api/channels/:channelId/streams', async (req, res) => { const params = req.params as any; From 37acc7d41189b934cb1aabe592179d96b40025d0 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 20:31:48 -0800 Subject: [PATCH 7/8] feat: Delete if not in use --- client/src/Streams.tsx | 6 ++++++ server/src/WebServer.ts | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/client/src/Streams.tsx b/client/src/Streams.tsx index 83d8115..cccbde6 100644 --- a/client/src/Streams.tsx +++ b/client/src/Streams.tsx @@ -63,6 +63,12 @@ export const StreamManager = () => { const providerRes : any = await providerReq.json(); + if (typeof providerRes.error !== 'undefined') { + alert(providerRes.error) + + return; + } + setStreams(streams.filter(i => i.id !== id)) } diff --git a/server/src/WebServer.ts b/server/src/WebServer.ts index 7d07ea0..333915d 100644 --- a/server/src/WebServer.ts +++ b/server/src/WebServer.ts @@ -81,6 +81,16 @@ export default class WebServer { fastify.delete('/api/streams/:streamId', async (req, res) => { const params = req.params as any; + const channelsUsing = await DatabaseEngine.AllSafe(`SELECT * FROM channel_source WHERE stream_id = ?;`, [params.streamId]) as ChannelSource[]; + + if (channelsUsing.length > 0) { + res.status(400); + + res.send({ error: 'Can not delete stream, stream has sources used by channels.' }) + + return; + } + await DatabaseEngine.RunSafe(`DELETE FROM streams WHERE id = ?;`, [params.streamId]); res.send(true); From 7ec58840de728e6d96675301dd7e2abc622f5832 Mon Sep 17 00:00:00 2001 From: Kalan Dominick Date: Tue, 21 Jan 2025 20:32:31 -0800 Subject: [PATCH 8/8] Version bump --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 08d0a1d..842fcc4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "piparr", - "version": "1.1.0", + "version": "1.2.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "piparr", - "version": "1.1.0", + "version": "1.2.0", "license": "Zlib", "dependencies": { "@fastify/static": "^7.0.4", diff --git a/package.json b/package.json index 41f66bb..5201484 100644 --- a/package.json +++ b/package.json @@ -31,7 +31,7 @@ "webpack-cli": "^5.1.4" }, "name": "piparr", - "version": "1.1.0", + "version": "1.2.0", "description": "M3U/IPTV Proxy Server", "main": "dist/index.js", "scripts": {