diff --git a/packages/nc-gui/components/smartsheet/toolbar/ExportSubActions.vue b/packages/nc-gui/components/smartsheet/toolbar/ExportSubActions.vue index 969c2e13bc59..935b137c43bd 100644 --- a/packages/nc-gui/components/smartsheet/toolbar/ExportSubActions.vue +++ b/packages/nc-gui/components/smartsheet/toolbar/ExportSubActions.vue @@ -1,83 +1,95 @@ @@ -89,20 +101,10 @@ const exportFile = async (exportType: ExportTypes) => {
- + CSV
- - -
- - - - - Excel -
- diff --git a/packages/nc-gui/plugins/api.ts b/packages/nc-gui/plugins/api.ts index 89fefac594f5..d2edbed78a5c 100644 --- a/packages/nc-gui/plugins/api.ts +++ b/packages/nc-gui/plugins/api.ts @@ -1,3 +1,5 @@ +import type { Api } from 'nocodb-sdk' + const apiPlugin = (nuxtApp) => { const { api } = useApi() @@ -7,7 +9,7 @@ const apiPlugin = (nuxtApp) => { declare module '#app' { interface NuxtApp { - $api: ReturnType + $api: Api } } diff --git a/packages/nocodb/src/controllers/data-alias-export.controller.spec.ts b/packages/nocodb/src/controllers/data-alias-export.controller.spec.ts deleted file mode 100644 index 7670b7a3c5c4..000000000000 --- a/packages/nocodb/src/controllers/data-alias-export.controller.spec.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { DataAliasExportController } from './data-alias-export.controller'; -import type { TestingModule } from '@nestjs/testing'; - -describe('DataAliasExportController', () => { - let controller: DataAliasExportController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [DataAliasExportController], - }).compile(); - - controller = module.get( - DataAliasExportController, - ); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/packages/nocodb/src/controllers/data-alias-export.controller.ts b/packages/nocodb/src/controllers/data-alias-export.controller.ts deleted file mode 100644 index 164445a8e6d5..000000000000 --- a/packages/nocodb/src/controllers/data-alias-export.controller.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { Controller, Get, Req, Res, UseGuards } from '@nestjs/common'; -import { Response } from 'express'; -import * as XLSX from 'xlsx'; -import { AppEvents } from 'nocodb-sdk'; -import { GlobalGuard } from '~/guards/global/global.guard'; -import { DatasService } from '~/services/datas.service'; -import { extractCsvData, extractXlsxData } from '~/helpers/dataHelpers'; -import { View } from '~/models'; -import { Acl } from '~/middlewares/extract-ids/extract-ids.middleware'; -import { DataApiLimiterGuard } from '~/guards/data-api-limiter.guard'; -import { TenantContext } from '~/decorators/tenant-context.decorator'; -import { NcContext, NcRequest } from '~/interface/config'; -import { AppHooksService } from '~/services/app-hooks/app-hooks.service'; - -@Controller() -@UseGuards(DataApiLimiterGuard, GlobalGuard) -export class DataAliasExportController { - constructor( - private datasService: DatasService, - private readonly appHooksService: AppHooksService, - ) {} - - @Get([ - '/api/v1/db/data/:orgs/:baseName/:tableName/export/excel', - '/api/v1/db/data/:orgs/:baseName/:tableName/views/:viewName/export/excel', - ]) - @Acl('exportExcel') - async excelDataExport( - @TenantContext() context: NcContext, - @Req() req: NcRequest, - @Res() res: Response, - ) { - const { model, view } = - await this.datasService.getViewAndModelFromRequestByAliasOrId( - context, - req, - ); - let targetView = view; - if (!targetView) { - targetView = await View.getDefaultView(context, model.id); - } - const { offset, elapsed, data } = await extractXlsxData( - context, - targetView, - req, - ); - const wb = XLSX.utils.book_new(); - XLSX.utils.book_append_sheet(wb, data, targetView.title); - const buf = XLSX.write(wb, { - type: req.query.encoding === 'base64' ? 'base64' : 'buffer', - bookType: 'xlsx', - }); - - this.appHooksService.emit(AppEvents.DATA_EXPORT, { - context, - req, - table: model, - view: targetView, - type: 'excel', - }); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - targetView.title, - )}-export.xlsx"`, - }); - res.end(buf); - } - - @Get([ - '/api/v1/db/data/:orgs/:baseName/:tableName/views/:viewName/export/csv', - '/api/v1/db/data/:orgs/:baseName/:tableName/export/csv', - ]) - @Acl('exportCsv') - async csvDataExport( - @TenantContext() context: NcContext, - @Req() req: NcRequest, - @Res() res: Response, - ) { - const { model, view } = - await this.datasService.getViewAndModelFromRequestByAliasOrId( - context, - req, - ); - let targetView = view; - if (!targetView) { - targetView = await View.getDefaultView(context, model.id); - } - const { offset, elapsed, data } = await extractCsvData( - context, - targetView, - req, - ); - - this.appHooksService.emit(AppEvents.DATA_EXPORT, { - context, - req, - table: model, - view: targetView, - type: 'csv', - }); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - targetView.title, - )}-export.csv"`, - }); - res.send(data); - } -} diff --git a/packages/nocodb/src/controllers/public-datas-export.controller.spec.ts b/packages/nocodb/src/controllers/public-datas-export.controller.spec.ts deleted file mode 100644 index ef682aac2977..000000000000 --- a/packages/nocodb/src/controllers/public-datas-export.controller.spec.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { Test } from '@nestjs/testing'; -import { PublicDatasExportService } from '../services/public-datas-export.service'; -import { PublicDatasExportController } from './public-datas-export.controller'; -import type { TestingModule } from '@nestjs/testing'; - -describe('PublicDatasExportController', () => { - let controller: PublicDatasExportController; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - controllers: [PublicDatasExportController], - providers: [PublicDatasExportService], - }).compile(); - - controller = module.get( - PublicDatasExportController, - ); - }); - - it('should be defined', () => { - expect(controller).toBeDefined(); - }); -}); diff --git a/packages/nocodb/src/controllers/public-datas-export.controller.ts b/packages/nocodb/src/controllers/public-datas-export.controller.ts deleted file mode 100644 index cbcf275baa46..000000000000 --- a/packages/nocodb/src/controllers/public-datas-export.controller.ts +++ /dev/null @@ -1,267 +0,0 @@ -import { - Controller, - Get, - Param, - Request, - Response, - UseGuards, -} from '@nestjs/common'; -import { isSystemColumn, ViewTypes } from 'nocodb-sdk'; -import * as XLSX from 'xlsx'; -import papaparse from 'papaparse'; -import { fromEntries } from '~/utils'; -import { nocoExecute } from '~/utils'; -import { NcError } from '~/helpers/catchError'; -import getAst from '~/helpers/getAst'; -import { serializeCellValue } from '~/helpers/dataHelpers'; -import { PublicDatasExportService } from '~/services/public-datas-export.service'; -import NcConnectionMgrv2 from '~/utils/common/NcConnectionMgrv2'; -import { Column, Model, Source, View } from '~/models'; -import { PublicApiLimiterGuard } from '~/guards/public-api-limiter.guard'; -import { TenantContext } from '~/decorators/tenant-context.decorator'; -import { NcContext } from '~/interface/config'; - -@UseGuards(PublicApiLimiterGuard) -@Controller() -export class PublicDatasExportController { - constructor( - private readonly publicDatasExportService: PublicDatasExportService, - ) {} - - @Get([ - '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/excel', - '/api/v2/public/shared-view/:publicDataUuid/rows/export/excel', - ]) - async exportExcel( - @TenantContext() context: NcContext, - @Request() req, - @Response() res, - @Param('publicDataUuid') publicDataUuid: string, - ) { - const view = await View.getByUUID(context, publicDataUuid); - if (!view) NcError.viewNotFound(publicDataUuid); - if ( - view.type !== ViewTypes.GRID && - view.type !== ViewTypes.KANBAN && - view.type !== ViewTypes.GALLERY && - view.type !== ViewTypes.CALENDAR && - view.type !== ViewTypes.MAP - ) - NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.invalidSharedViewPassword(); - } - - // check if download is allowed, in general it's called as CSV download - if (!view.meta?.allowCSVDownload) { - NcError.forbidden('Download is not allowed for this view'); - } - - const model = await view.getModelWithInfo(context); - - await view.getColumns(context); - const modelColumnMap = model.columns.reduce((mapObj, cur) => { - mapObj[cur.id] = cur; - return mapObj; - }, {}); - - const { offset, dbRows, elapsed } = await this.getDbRows( - context, - model, - view, - req, - ); - - let fields = req.query.fields as string[]; - const allowedColumns = view.columns - .filter((c) => c.show) - .map((k) => modelColumnMap[k.fk_column_id]) - .filter((column) => !isSystemColumn(column) || view.show_system_fields) - .map((k) => k.title); - if (!fields || fields.length === 0 || !Array.isArray(fields)) { - fields = allowedColumns; - } else { - fields = fields.filter((field) => allowedColumns.includes(field)); - } - - const data = XLSX.utils.json_to_sheet( - dbRows.map((o: Record) => - fromEntries(fields.map((f) => [f, o[f]])), - ), - { header: fields }, - ); - - const wb = XLSX.utils.book_new(); - - XLSX.utils.book_append_sheet(wb, data, view.title); - const buf = XLSX.write(wb, { - type: req.query.encoding === 'base64' ? 'base64' : 'buffer', - bookType: 'xlsx', - }); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - view.title, - )}-export.xlsx"`, - }); - res.end(buf); - } - - @Get([ - '/api/v1/db/public/shared-view/:publicDataUuid/rows/export/csv', - '/api/v2/public/shared-view/:publicDataUuid/rows/export/csv', - ]) - async exportCsv( - @TenantContext() context: NcContext, - @Request() req, - @Response() res, - ) { - const view = await View.getByUUID(context, req.params.publicDataUuid); - const fields = req.query.fields; - - if (!view) NcError.viewNotFound(req.params.publicDataUuid); - if ( - view.type !== ViewTypes.GRID && - view.type !== ViewTypes.KANBAN && - view.type !== ViewTypes.GALLERY && - view.type !== ViewTypes.CALENDAR && - view.type !== ViewTypes.MAP - ) - NcError.notFound('Not found'); - - if (view.password && view.password !== req.headers?.['xc-password']) { - NcError.invalidSharedViewPassword(); - } - - // check if download is allowed - if (!view.meta?.allowCSVDownload) { - NcError.forbidden('Download is not allowed for this view'); - } - - const model = await view.getModelWithInfo(context); - await view.getColumns(context); - - const { offset, dbRows, elapsed } = await this.getDbRows( - context, - model, - view, - req, - ); - - const data = papaparse.unparse( - { - fields: model.columns - .sort((c1, c2) => - Array.isArray(fields) - ? fields.indexOf(c1.title as any) - - fields.indexOf(c2.title as any) - : 0, - ) - .filter( - (c) => - !fields || - !Array.isArray(fields) || - fields.includes(c.title as any), - ) - .map((c) => c.title), - data: dbRows, - }, - { - escapeFormulae: true, - }, - ); - - res.set({ - 'Access-Control-Expose-Headers': 'nc-export-offset', - 'nc-export-offset': offset, - 'nc-export-elapsed-time': elapsed, - 'Content-Disposition': `attachment; filename="${encodeURI( - view.title, - )}-export.csv"`, - }); - res.send(data); - } - - async getDbRows(@TenantContext() context: NcContext, model, view: View, req) { - view.model.columns = view.columns - .filter((c) => c.show) - .map( - (c) => - new Column({ - ...c, - ...view.model.columnsById[c.fk_column_id], - } as any), - ) - .filter((column) => !isSystemColumn(column) || view.show_system_fields); - - if (!model) NcError.notFound('Table not found'); - - const listArgs: any = { ...req.query }; - try { - listArgs.filterArr = JSON.parse(listArgs.filterArrJson); - } catch (e) {} - try { - listArgs.sortArr = JSON.parse(listArgs.sortArrJson); - } catch (e) {} - - const source = await Source.get(context, model.source_id); - const baseModel = await Model.getBaseModelSQL(context, { - id: model.id, - viewId: view?.id, - dbDriver: await NcConnectionMgrv2.get(source), - }); - - const { ast } = await getAst(context, { - query: req.query, - model, - view, - includePkByDefault: false, - }); - - let offset = +req.query.offset || 0; - const limit = 100; - // const size = +process.env.NC_EXPORT_MAX_SIZE || 1024; - const timeout = +process.env.NC_EXPORT_MAX_TIMEOUT || 5000; - const dbRows = []; - const startTime = process.hrtime(); - let elapsed, temp; - - for ( - elapsed = 0; - elapsed < timeout; - offset += limit, - temp = process.hrtime(startTime), - elapsed = temp[0] * 1000 + temp[1] / 1000000 - ) { - const rows = await nocoExecute( - ast, - await baseModel.list({ ...listArgs, offset, limit }), - {}, - listArgs, - ); - - if (!rows?.length) { - offset = -1; - break; - } - - for (const row of rows) { - const dbRow = { ...row }; - - for (const column of view.model.columns) { - dbRow[column.title] = await serializeCellValue(context, { - value: row[column.title], - column, - siteUrl: req.ncSiteUrl, - }); - } - dbRows.push(dbRow); - } - } - return { offset, dbRows, elapsed }; - } -} diff --git a/packages/nocodb/src/interface/Jobs.ts b/packages/nocodb/src/interface/Jobs.ts index 682fd45095f1..1a97e448aa75 100644 --- a/packages/nocodb/src/interface/Jobs.ts +++ b/packages/nocodb/src/interface/Jobs.ts @@ -35,6 +35,7 @@ export enum JobTypes { HandleWebhook = 'handle-webhook', CleanUp = 'clean-up', DataExport = 'data-export', + DataExportCleanUp = 'data-export-clean-up', ThumbnailGenerator = 'thumbnail-generator', AttachmentCleanUp = 'attachment-clean-up', InitMigrationJobs = 'init-migration-jobs', diff --git a/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts index 6e747d5077c1..582758f30be3 100644 --- a/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts +++ b/packages/nocodb/src/modules/jobs/fallback/fallback-queue.service.ts @@ -1,6 +1,7 @@ import { forwardRef, Inject, Injectable } from '@nestjs/common'; import PQueue from 'p-queue'; import Emittery from 'emittery'; +import { CronExpressionParser } from 'cron-parser'; import { JobsEventService } from '~/modules/jobs/jobs-event.service'; import { JobStatus } from '~/interface/Jobs'; import { JobsMap } from '~/modules/jobs/jobs-map.service'; @@ -10,6 +11,8 @@ export interface Job { name: string; status: string; data: any; + repeat?: { cron: string }; + delay?: number; } @Injectable() @@ -102,7 +105,11 @@ export class QueueService { QueueService.queueIdCounter = index; } - add(name: string, data: any, opts?: { jobId?: string; delay?: number }) { + add( + name: string, + data: any, + opts?: { jobId?: string; delay?: number; repeat?: { cron: string } }, + ) { const id = opts?.jobId || `${this.queueIndex++}`; const existingJob = this.queueMemory.find((q) => q.id === id); @@ -131,6 +138,11 @@ export class QueueService { }; if (existingJob) { + // if job is already in memory and has a repeat, return the job + if (existingJob.repeat) { + return existingJob; + } + if (existingJob.status !== JobStatus.WAITING) { existingJob.status = JobStatus.WAITING; } @@ -141,11 +153,51 @@ export class QueueService { name, status: JobStatus.WAITING, data, + repeat: opts?.repeat, + delay: opts?.delay, ...helperFns(), }; } - if (opts?.delay) { + if (opts?.repeat) { + // Initialize recurring job execution based on cron pattern + const scheduleNextExecution = () => { + try { + const cron = CronExpressionParser.parse(opts.repeat.cron); + const nextExecutionTime = cron.next().toDate(); + const delayMs = Math.max(0, nextExecutionTime.getTime() - Date.now()); + + const recurringJobId = `${id}-${Date.now()}`; + const childJob = { ...job, id: recurringJobId }; + + setTimeout(() => { + this.queueMemory.push(childJob); + // Execute the current job + this.queue + .add(() => this.jobWrapper(childJob)) + .then(() => { + // Schedule the next execution only after current job completes + this.queue.add(() => scheduleNextExecution()); + }) + .catch((error) => { + console.error( + `Failed to schedule recurring job ${name}:`, + error, + ); + // Try to reschedule despite the error + this.queue.add(() => scheduleNextExecution()); + }); + }, delayMs); + } catch (error) { + console.error(`Invalid cron expression for job ${name}:`, error); + } + }; + + this.queueMemory.push(job); + + // Start the recurring job process + scheduleNextExecution(); + } else if (opts?.delay) { if (!existingJob) { this.queueMemory.push(job); } diff --git a/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts b/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts index c419d989bbae..4bccf618fac4 100644 --- a/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts +++ b/packages/nocodb/src/modules/jobs/fallback/jobs.service.ts @@ -16,6 +16,16 @@ export class JobsService implements OnModuleInit { constructor(private readonly fallbackQueueService: QueueService) {} async onModuleInit() { + await this.fallbackQueueService.add( + JobTypes.DataExportCleanUp, + {}, + { + jobId: JobTypes.DataExportCleanUp, + // run every 5 hours + repeat: { cron: '* * * * *' }, + }, + ); + await this.add(JobTypes.InitMigrationJobs, {}); } diff --git a/packages/nocodb/src/modules/jobs/jobs-map.service.ts b/packages/nocodb/src/modules/jobs/jobs-map.service.ts index bf7b92b446a8..28bbefa2cdcd 100644 --- a/packages/nocodb/src/modules/jobs/jobs-map.service.ts +++ b/packages/nocodb/src/modules/jobs/jobs-map.service.ts @@ -10,6 +10,7 @@ import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-gener import { AttachmentCleanUpProcessor } from '~/modules/jobs/jobs/attachment-clean-up/attachment-clean-up'; import { InitMigrationJobs } from '~/modules/jobs/migration-jobs/init-migration-jobs'; import { UseWorkerProcessor } from '~/modules/jobs/jobs/use-worker/use-worker.processor'; +import { DataExportCleanUpProcessor } from '~/modules/jobs/jobs/data-export-clean-up/data-export-clean-up.processor'; import { JobTypes } from '~/interface/Jobs'; @Injectable() @@ -26,6 +27,7 @@ export class JobsMap { protected readonly attachmentCleanUpProcessor: AttachmentCleanUpProcessor, protected readonly initMigrationJobs: InitMigrationJobs, protected readonly useWorkerProcessor: UseWorkerProcessor, + protected readonly dataExportCleanUpProcessor: DataExportCleanUpProcessor, ) {} protected get _jobMap(): { @@ -65,6 +67,9 @@ export class JobsMap { [JobTypes.DataExport]: { this: this.dataExportProcessor, }, + [JobTypes.DataExportCleanUp]: { + this: this.dataExportCleanUpProcessor, + }, [JobTypes.ThumbnailGenerator]: { this: this.thumbnailGeneratorProcessor, }, diff --git a/packages/nocodb/src/modules/jobs/jobs.module.ts b/packages/nocodb/src/modules/jobs/jobs.module.ts index 1b354b3d0b18..53bff8a7f7e1 100644 --- a/packages/nocodb/src/modules/jobs/jobs.module.ts +++ b/packages/nocodb/src/modules/jobs/jobs.module.ts @@ -23,6 +23,8 @@ import { SourceDeleteProcessor } from '~/modules/jobs/jobs/source-delete/source- import { WebhookHandlerProcessor } from '~/modules/jobs/jobs/webhook-handler/webhook-handler.processor'; import { DataExportProcessor } from '~/modules/jobs/jobs/data-export/data-export.processor'; import { DataExportController } from '~/modules/jobs/jobs/data-export/data-export.controller'; +import { DataExportCleanUpProcessor } from '~/modules/jobs/jobs/data-export-clean-up/data-export-clean-up.processor'; +import { PublicDataExportController } from '~/modules/jobs/jobs/data-export/public-data-export.controller'; import { ThumbnailGeneratorProcessor } from '~/modules/jobs/jobs/thumbnail-generator/thumbnail-generator.processor'; import { AttachmentCleanUpProcessor } from '~/modules/jobs/jobs/attachment-clean-up/attachment-clean-up'; import { UseWorkerProcessor } from '~/modules/jobs/jobs/use-worker/use-worker.processor'; @@ -84,6 +86,7 @@ export const JobsModuleMetadata = { SourceCreateController, SourceDeleteController, DataExportController, + PublicDataExportController, ] : []), ], @@ -110,6 +113,7 @@ export const JobsModuleMetadata = { SourceDeleteProcessor, WebhookHandlerProcessor, DataExportProcessor, + DataExportCleanUpProcessor, ThumbnailGeneratorProcessor, AttachmentCleanUpProcessor, UseWorkerProcessor, diff --git a/packages/nocodb/src/modules/jobs/jobs/data-export-clean-up/data-export-clean-up.processor.ts b/packages/nocodb/src/modules/jobs/jobs/data-export-clean-up/data-export-clean-up.processor.ts new file mode 100644 index 000000000000..149ecd6253f6 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/jobs/data-export-clean-up/data-export-clean-up.processor.ts @@ -0,0 +1,110 @@ +import debug from 'debug'; +import PQueue from 'p-queue'; +import { Injectable } from '@nestjs/common'; +import moment from 'moment'; +import type { Job } from 'bull'; +import NcPluginMgrv2 from '~/helpers/NcPluginMgrv2'; + +@Injectable() +export class DataExportCleanUpProcessor { + private readonly debugLog = debug('nc:jobs:data-export-clean-up'); + + async job(job: Job) { + this.debugLog(`Job started for ${job.id}`); + + const queue = new PQueue({ concurrency: 1 }); + + try { + const storageAdapter = await NcPluginMgrv2.storageAdapter(); + const cutoffDate = moment().subtract(4, 'hours'); + + // Convert cutoff date to YYYY-MM-DD format to match folder structure + const cutoffDateFormatted = cutoffDate.format('YYYY-MM-DD'); + + // The pattern will match all files in data-export folders that are older than the cutoff date + const globPattern = `nc/uploads/data-export/**`; + + const fileStream = await storageAdapter.scanFiles(globPattern); + + // Track statistics for logging + let scannedCount = 0; + let deletedCount = 0; + let errorCount = 0; + + return new Promise((resolve, reject) => { + fileStream.on('error', (error) => { + this.debugLog(`Error scanning files: ${error.message}`); + reject(error); + }); + + fileStream.on('data', (filePath) => { + scannedCount++; + + // Add file processing task to the queue + queue + .add(async () => { + try { + // Skip if not a data export file + if ( + !filePath || + typeof filePath !== 'string' || + !filePath.startsWith('nc/uploads/data-export') + ) { + return; + } + + // Extract date from file path (format: nc/uploads/data-export/YYYY-MM-DD/HH/...) + const pathParts = filePath.split('/'); + const dateIndex = pathParts.indexOf('data-export') + 1; + + if (dateIndex >= pathParts.length || !pathParts[dateIndex]) { + return; + } + + const folderDate = pathParts[dateIndex]; + + // Check if folder date is before cutoff date + if (folderDate && folderDate < cutoffDateFormatted) { + await storageAdapter.fileDelete(filePath); + deletedCount++; + this.debugLog(`Deleted old export file: ${filePath}`); + } + } catch (e) { + errorCount++; + this.debugLog( + `Error processing file ${filePath}: ${e.message}`, + ); + // Don't rethrow to prevent queue from stopping + } + }) + .catch((err) => { + errorCount++; + this.debugLog( + `Queue error processing file ${filePath}: ${err.message}`, + ); + }); + }); + + fileStream.on('end', async () => { + try { + // Wait for all queued tasks to complete + await queue.onIdle(); + this.debugLog( + `Data export cleanup completed: Scanned ${scannedCount} files, deleted ${deletedCount} files, encountered ${errorCount} errors`, + ); + this.debugLog(`Job completed for ${job.id}`); + resolve(true); + } catch (err) { + this.debugLog( + `Error waiting for queue to complete: ${err.message}`, + ); + reject(err); + } + }); + }); + } catch (e) { + this.debugLog(`Job failed: ${e.message}`); + throw e; + } + } +} diff --git a/packages/nocodb/src/modules/jobs/jobs/data-export/public-data-export.controller.ts b/packages/nocodb/src/modules/jobs/jobs/data-export/public-data-export.controller.ts new file mode 100644 index 000000000000..0ee53be584c7 --- /dev/null +++ b/packages/nocodb/src/modules/jobs/jobs/data-export/public-data-export.controller.ts @@ -0,0 +1,76 @@ +import { + Body, + Controller, + HttpCode, + Inject, + Param, + Post, + Req, + UseGuards, +} from '@nestjs/common'; +import { ViewTypes } from 'nocodb-sdk'; +import type { DataExportJobData } from '~/interface/Jobs'; +import { BasesService } from '~/services/bases.service'; +import { View } from '~/models'; +import { JobTypes } from '~/interface/Jobs'; +import { IJobsService } from '~/modules/jobs/jobs-service.interface'; +import { TenantContext } from '~/decorators/tenant-context.decorator'; +import { NcContext, NcRequest } from '~/interface/config'; +import { NcError } from '~/helpers/catchError'; +import { PublicApiLimiterGuard } from '~/guards/public-api-limiter.guard'; + +@Controller() +@UseGuards(PublicApiLimiterGuard) +export class PublicDataExportController { + constructor( + @Inject('JobsService') protected readonly jobsService: IJobsService, + protected readonly basesService: BasesService, + ) {} + + @Post(['/api/v2/public/export/:publicDataUuid/:exportAs']) + @HttpCode(200) + async exportModelData( + @TenantContext() context: NcContext, + @Req() req: NcRequest, + @Param('publicDataUuid') publicDataUuid: string, + @Param('exportAs') exportAs: 'csv' | 'json' | 'xlsx', + @Body() options: DataExportJobData['options'], + ) { + const view = await View.getByUUID(context, publicDataUuid); + + if (!view) NcError.viewNotFound(publicDataUuid); + if ( + view.type !== ViewTypes.GRID && + view.type !== ViewTypes.KANBAN && + view.type !== ViewTypes.GALLERY && + view.type !== ViewTypes.CALENDAR && + view.type !== ViewTypes.MAP + ) + NcError.notFound('Not found'); + + if (view.password && view.password !== req.headers?.['xc-password']) { + NcError.invalidSharedViewPassword(); + } + + // check if download is allowed + if (!view.meta?.allowCSVDownload) { + NcError.forbidden('Download is not allowed for this view'); + } + + if (!view) NcError.viewNotFound(publicDataUuid); + + const job = await this.jobsService.add(JobTypes.DataExport, { + context, + options, + modelId: view.fk_model_id, + viewId: view.id, + user: req.user, + exportAs, + ncSiteUrl: req.ncSiteUrl, + }); + + return { + id: job.id, + }; + } +} diff --git a/packages/nocodb/src/modules/jobs/redis/jobs.service.ts b/packages/nocodb/src/modules/jobs/redis/jobs.service.ts index 6c889454f165..6b17f2c7062e 100644 --- a/packages/nocodb/src/modules/jobs/redis/jobs.service.ts +++ b/packages/nocodb/src/modules/jobs/redis/jobs.service.ts @@ -28,6 +28,18 @@ export class JobsService implements OnModuleInit { await this.jobsQueue.pause(true); } + await this.jobsQueue.add( + { + jobName: JobTypes.DataExportCleanUp, + context: {}, + }, + { + jobId: JobTypes.DataExportCleanUp, + // run every 5 hours + repeat: { cron: '0 */5 * * *' }, + }, + ); + await this.toggleQueue(); JobsRedis.workerCallbacks[InstanceCommands.RESUME_LOCAL] = async () => { diff --git a/packages/nocodb/src/modules/noco.module.ts b/packages/nocodb/src/modules/noco.module.ts index 6242fc769ac7..6ad3c8502753 100644 --- a/packages/nocodb/src/modules/noco.module.ts +++ b/packages/nocodb/src/modules/noco.module.ts @@ -116,7 +116,6 @@ import { InternalController } from '~/controllers/internal.controller'; /* Datas */ import { BulkDataAliasController } from '~/controllers/bulk-data-alias.controller'; import { CalendarDatasController } from '~/controllers/calendars-datas.controller'; -import { DataAliasExportController } from '~/controllers/data-alias-export.controller'; import { DataAliasNestedController } from '~/controllers/data-alias-nested.controller'; import { DataAliasController } from '~/controllers/data-alias.controller'; import { DataTableController } from '~/controllers/data-table.controller'; @@ -124,7 +123,6 @@ import { DatasController } from '~/controllers/datas.controller'; import { IntegrationsController } from '~/controllers/integrations.controller'; import { OldDatasController } from '~/controllers/old-datas/old-datas.controller'; import { OldDatasService } from '~/controllers/old-datas/old-datas.service'; -import { PublicDatasExportController } from '~/controllers/public-datas-export.controller'; import { PublicDatasController } from '~/controllers/public-datas.controller'; import { BaseUsersV3Controller } from '~/controllers/v3/base-users-v3.controller'; import { BasesV3Controller } from '~/controllers/v3/bases-v3.controller'; @@ -241,10 +239,8 @@ export const nocoModuleMetadata = { BulkDataAliasController, DataAliasController, DataAliasNestedController, - DataAliasExportController, OldDatasController, PublicDatasController, - PublicDatasExportController, Datav3Controller, ] : []), diff --git a/packages/nocodb/src/schema/swagger-v2.json b/packages/nocodb/src/schema/swagger-v2.json index 02c9204329a4..2d7f46adc843 100644 --- a/packages/nocodb/src/schema/swagger-v2.json +++ b/packages/nocodb/src/schema/swagger-v2.json @@ -9104,64 +9104,6 @@ "description": "List all nested list data in a given shared view" } }, - "/api/v2/public/shared-view/{sharedViewUuid}/rows/export/{type}": { - "parameters": [ - { - "schema": { - "type": "string", - "example": "76d44b86-bc65-4500-8956-ab512c80ab25" - }, - "name": "sharedViewUuid", - "in": "path", - "required": true, - "description": "Shared View UUID" - }, - { - "schema": { - "type": "string", - "enum": [ - "csv", - "excel" - ], - "example": "csv" - }, - "name": "type", - "in": "path", - "required": true, - "description": "Export Type" - } - ], - "get": { - "summary": "Export Rows in Share View", - "operationId": "public-csv-export", - "description": "Export all rows in Share View in a CSV / Excel Format", - "wrapped": true, - "responses": { - "200": { - "description": "OK", - "content": { - "application/octet-stream": { - "schema": {} - } - }, - "headers": { - "nc-export-offset": { - "schema": { - "type": "integer" - }, - "description": "The starting offset of the next set of rows" - } - } - }, - "400": { - "$ref": "#/components/responses/BadRequest" - } - }, - "tags": [ - "Public", "Internal" - ] - } - }, "/api/v2/public/shared-view/{sharedViewUuid}/nested/{columnName}": { "parameters": [ { @@ -12926,6 +12868,50 @@ "$ref": "#/components/parameters/xc-token" } ] + }, + "/api/v2/public/export/{publicDataUuid}/{exportAs}": { + "post": { + "summary": "Trigger export as job", + "operationId": "public-export-data", + "description": "Trigger export as job", + "tags": [ + "Public" + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "parameters": [ + { + "schema": { + "$ref": "#/components/schemas/Id", + "example": "vw124dflkcvasewh", + "type": "string" + }, + "name": "publicDataUuid", + "in": "path", + "required": true, + "description": "Unique View ID" + }, + { + "schema": { + "type": "string", + "enum": [ + "csv" + ] + }, + "name": "exportAs", + "in": "path", + "required": true, + "description": "Export as format" + } + ] } }, "components": { diff --git a/packages/nocodb/src/schema/swagger.json b/packages/nocodb/src/schema/swagger.json index 82f018eacda7..0764b6475c7a 100644 --- a/packages/nocodb/src/schema/swagger.json +++ b/packages/nocodb/src/schema/swagger.json @@ -12411,170 +12411,6 @@ ] } }, - "/api/v1/db/data/{orgs}/{baseName}/{tableName}/views/{viewName}/export/{type}": { - "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "orgs", - "in": "path", - "required": true, - "description": "Organisation Name. Currently `noco` will be used." - }, - { - "schema": { - "type": "string" - }, - "name": "baseName", - "in": "path", - "required": true, - "description": "Base Name" - }, - { - "schema": { - "type": "string" - }, - "name": "tableName", - "in": "path", - "required": true, - "description": "Table Name" - }, - { - "schema": { - "type": "string" - }, - "name": "viewName", - "in": "path", - "required": true - }, - { - "schema": { - "type": "string", - "enum": [ - "csv", - "excel" - ] - }, - "name": "type", - "in": "path", - "required": true - } - ], - "get": { - "summary": "Export Table View Rows", - "operationId": "db-view-row-export", - "description": "Export Table View Rows by CSV or Excel", - "tags": [ - "DB View Row" - ], - "wrapped": true, - "responses": { - "200": { - "description": "OK", - "content": { - "application/octet-stream": { - "schema": {} - } - }, - "headers": { - "nc-export-offset": { - "schema": { - "type": "integer" - }, - "description": "The starting offset of the next set of rows" - } - } - }, - "400": { - "$ref": "#/components/responses/BadRequest" - } - }, - "parameters": [ - { - "$ref": "#/components/parameters/xc-auth" - } - ] - } - }, - "/api/v1/db/data/{orgs}/{baseName}/{tableName}/export/{type}": { - "parameters": [ - { - "schema": { - "type": "string" - }, - "name": "orgs", - "in": "path", - "required": true, - "description": "Organisation Name. Currently `noco` will be used." - }, - { - "schema": { - "type": "string" - }, - "name": "baseName", - "in": "path", - "required": true, - "description": "Base Name" - }, - { - "schema": { - "type": "string" - }, - "name": "tableName", - "in": "path", - "required": true, - "description": "Table Name" - }, - { - "schema": { - "type": "string", - "enum": [ - "csv", - "excel" - ] - }, - "name": "type", - "in": "path", - "required": true - } - ], - "get": { - "summary": "Export Table View Rows", - "operationId": "db-table-row-csv-export", - "description": "Export Table View Rows by CSV or Excel", - "tags": [ - "DB Table Row" - ], - "wrapped": true, - "responses": { - "200": { - "description": "OK", - "content": { - "application/octet-stream": { - "schema": {} - } - }, - "headers": { - "nc-export-offset": { - "schema": { - "type": "integer" - }, - "description": "The starting offset of the next set of rows" - } - } - }, - "400": { - "$ref": "#/components/responses/BadRequest" - } - }, - "parameters": [ - { - "$ref": "#/components/parameters/xc-auth" - } - ] - } - }, "/api/v1/db/data/{orgs}/{baseName}/{tableName}/{rowId}/{relationType}/{columnName}": { "parameters": [ { @@ -14301,64 +14137,6 @@ "description": "List all nested list data in a given shared view" } }, - "/api/v1/db/public/shared-view/{sharedViewUuid}/rows/export/{type}": { - "parameters": [ - { - "schema": { - "type": "string", - "example": "76d44b86-bc65-4500-8956-ab512c80ab25" - }, - "name": "sharedViewUuid", - "in": "path", - "required": true, - "description": "Shared View UUID" - }, - { - "schema": { - "type": "string", - "enum": [ - "csv", - "excel" - ], - "example": "csv" - }, - "name": "type", - "in": "path", - "required": true, - "description": "Export Type" - } - ], - "get": { - "summary": "Export Rows in Share View", - "operationId": "public-csv-export", - "description": "Export all rows in Share View in a CSV / Excel Format", - "wrapped": true, - "responses": { - "200": { - "description": "OK", - "content": { - "application/octet-stream": { - "schema": {} - } - }, - "headers": { - "nc-export-offset": { - "schema": { - "type": "integer" - }, - "description": "The starting offset of the next set of rows" - } - } - }, - "400": { - "$ref": "#/components/responses/BadRequest" - } - }, - "tags": [ - "Public" - ] - } - }, "/api/v1/db/public/shared-view/{sharedViewUuid}/nested/{columnName}": { "parameters": [ { @@ -19419,6 +19197,50 @@ } ] }, + "/api/v2/public/export/{publicDataUuid}/{exportAs}": { + "post": { + "summary": "Trigger export as job", + "operationId": "public-export-data", + "description": "Trigger export as job", + "tags": [ + "Public" + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "type": "object" + } + } + } + } + }, + "parameters": [ + { + "schema": { + "$ref": "#/components/schemas/Id", + "example": "vw124dflkcvasewh", + "type": "string" + }, + "name": "publicDataUuid", + "in": "path", + "required": true, + "description": "Unique View ID" + }, + { + "schema": { + "type": "string", + "enum": [ + "csv" + ] + }, + "name": "exportAs", + "in": "path", + "required": true, + "description": "Export as format" + } + ] + }, "/api/v2/meta/hooks/{hookId}/trigger/{rowId}": { "parameters": [ { diff --git a/packages/nocodb/src/services/api-docs/swagger/templates/paths.ts b/packages/nocodb/src/services/api-docs/swagger/templates/paths.ts index a52d8807aeb9..b2b1e0a67303 100644 --- a/packages/nocodb/src/services/api-docs/swagger/templates/paths.ts +++ b/packages/nocodb/src/services/api-docs/swagger/templates/paths.ts @@ -450,29 +450,6 @@ export const getModelPaths = async ( : {}), } : {}), - [`/api/v1/db/data/${ctx.orgs}/${ctx.baseName}/${ctx.tableName}/export/{type}`]: - { - parameters: [exportTypeParam], - get: { - summary: 'Rows export', - operationId: `${ctx.tableName.toLowerCase()}-csv-export`, - description: - 'Export all the records from a table.Currently we are only supports `csv` export.', - tags: [ctx.tableName], - responses: { - '200': { - description: 'OK', - content: { - 'application/octet-stream': { - schema: {}, - }, - }, - headers: csvExportResponseHeader, - }, - }, - parameters: [csvExportOffsetParam], - }, - }, }); export const getViewPaths = async ( @@ -633,29 +610,6 @@ export const getViewPaths = async ( }, } : {}), - [`/api/v1/db/data/${ctx.orgs}/${ctx.baseName}/${ctx.tableName}/views/${ctx.viewName}/export/{type}`]: - { - parameters: [exportTypeParam], - get: { - summary: `${ctx.viewName} export`, - operationId: `${ctx.tableName}-${ctx.viewName}-row-export`, - description: - 'Export all the records from a table view. Currently we are only supports `csv` export.', - tags: [`${ctx.viewName} ( ${ctx.tableName} grid )`], - responses: { - '200': { - description: 'OK', - content: { - 'application/octet-stream': { - schema: {}, - }, - }, - headers: csvExportResponseHeader, - }, - }, - parameters: [], - }, - }, }); function getPaginatedResponseType(type: string) {