/**
* The Content Manager is mainly in charge of storing and retrieving
* all the content that is stored and known by the bot. The content includes (but is not limited to)
* the messages that the bot sends to users.
* @see {@link https://botpress.io/docs/10.0/getting_started/trivia_content/}
* @namespace ContentManager
* @example
* bp.contentManager
*/
import path from 'path'
import fs from 'fs'
import _ from 'lodash'
import Promise from 'bluebird'
import glob from 'glob'
import mkdirp from 'mkdirp'
import nanoid from 'nanoid'
import helpers from '../database/helpers'
import { getInMemoryDb } from '../util'
const getNewItemId = category => {
const prefix = (category.renderer || category.id).replace(/^#/, '')
return `${prefix}-${nanoid(6)}`
}
const prepareDb = async () => {
const knex = getInMemoryDb()
// NB! This is in-memory temprorary database
// It is freshly created so we know there are no tables
// We also use camelCased columns for convenience
await knex.schema.createTable('content_items', table => {
table.string('id').primary()
table.text('data')
table.text('formData')
table.text('metadata')
table.string('categoryId')
table.text('previewText')
table.string('createdBy')
table.timestamp('createdOn')
})
return knex
}
const defaults = {
contentDir: './content',
contentDataDir: './content_data'
}
module.exports = async ({ botfile, projectLocation, logger, ghostManager }) => {
const categories = []
const categoryById = {}
const fileById = {}
const getItemProviders = {}
const contentDir = path.resolve(projectLocation, botfile.contentDir || defaults.contentDir)
const contentDataDir = path.resolve(projectLocation, botfile.contentDataDir || defaults.contentDataDir)
const knex = await prepareDb()
const transformItemDbToApi = item => {
if (!item) {
return item
}
return {
...item,
data: JSON.parse(item.data),
formData: JSON.parse(item.formData),
metadata: (item.metadata || '').split('|').filter(i => i.length > 0)
}
}
const transformItemApiToDb = item => {
if (!item) {
return item
}
const result = { ...item }
if ('formData' in item) {
result.formData = JSON.stringify(item.formData)
}
if ('data' in item) {
result.data = JSON.stringify(item.data)
}
if ('metadata' in item) {
result.metadata = '|' + (item.metadata || []).filter(i => !!i).join('|') + '|'
}
return result
}
/**
* @typedef {Object} ContentManager~Element
* @memberOf ContentManager
*/
/**
* @typedef {Object} ContentManager~CategorySchema
* @memberOf ContentManager
* @prop {Object} json The JSONSchema
* @prop {String} ui The UI JSONSchema
* @property {String} description
* @property {String} renderer The name of the Content Renderer
*/
/**
* @typedef {Object} ContentManager~Category
* @memberOf ContentManager
* @prop {String} id
* @prop {String} title
* @property {String} description
* @property {Number} count The number of elements in that category
* @property {ContentManager~CategorySchema} schema
*/
/**
* Returns the elements of a given category
* @param {String} categoryId The category, for example `text` or `trivia`.
* @param {Number} [options.from=0] Pagination parameter (where to start)
* @param {Number} [options.count=50] Pagination parameter (how many elements to return)
* @param {String} [options.searchTerm=] Only return the elements containing this term
* @param {Array.<String>} [options.orderBy=['createdOn']] A list of properties to order the elements by.
* @return {ContentManager~Element[]}
* @public
* @memberOf! ContentManager
*/
const listCategoryItems = async (categoryId, { from = 0, count = 50, searchTerm, orderBy = ['createdOn'] } = {}) => {
let query = knex('content_items')
if (categoryId) {
query = query.where({ categoryId })
}
if (searchTerm) {
query = query.where(function() {
this.where('metadata', 'like', `%${searchTerm}%`)
.orWhere('formData', 'like', `%${searchTerm}%`)
.orWhere('id', 'like', `%${searchTerm}%`)
})
}
const items = await query
.orderBy(...orderBy)
.offset(from)
.limit(count)
.then()
return items.map(transformItemDbToApi)
}
const dumpDataToFile = async categoryId => {
// TODO Do paging here and dump *everything*
const items = (await listCategoryItems(categoryId, { count: 10000 })).map(item =>
_.pick(item, 'id', 'formData', 'createdBy', 'createdOn')
)
await ghostManager.upsertFile(contentDataDir, fileById[categoryId], JSON.stringify(items, null, 2))
}
const dumpAllDataToFiles = () => Promise.map(categories, ({ id }) => dumpDataToFile(id))
/**
* Get the schema for a given category
* @param {String} categoryId [description]
* @return {ContentManager~CategorySchema}
*/
const getCategorySchema = categoryId => {
const category = _.find(categories, { id: categoryId })
if (category == null) {
return null
}
return {
json: category.jsonSchema,
ui: category.uiSchema,
title: category.title,
description: category.description,
renderer: category.renderer
}
}
/**
* Returns all the categories
* @public
* @return {ContentManager~Category[]}
* @memberOf! ContentManager
*/
const listAvailableCategories = () =>
Promise.map(categories, async category => {
const count = await knex('content_items')
.where({ categoryId: category.id })
.count('* as count')
.get(0)
.then(row => (row && Number(row.count)) || 0)
return {
id: category.id,
title: category.title,
description: category.description,
count,
schema: getCategorySchema(category.id)
}
})
const resolveRefs = async data => {
if (!data) {
return data
}
if (Array.isArray(data)) {
return Promise.map(data, resolveRefs)
}
if (_.isObject(data)) {
return Promise.props(_.mapValues(data, resolveRefs))
}
if (_.isString(data)) {
const m = data.match(/^##ref\((.*)\)$/)
if (!m) {
return data
}
return knex('content_items')
.select('formData')
.where('id', m[1])
.then(result => {
if (!result || !result.length) {
throw new Error(`Error resolving reference: ID ${m[1]} not found.`)
}
return JSON.parse(result[0].formData)
})
.then(resolveRefs)
}
return data
}
const computeData = async (categoryId, formData) => {
const category = categoryById[categoryId]
if (!category) {
throw new Error(`Unknown category ${categoryId}`)
}
return !category.computeData ? formData : category.computeData(formData, computeData)
}
const computeMetadata = async (categoryId, formData) => {
const category = categoryById[categoryId]
if (!category) {
throw new Error(`Unknown category ${categoryId}`)
}
return !category.computeMetadata ? [] : category.computeMetadata(formData, computeMetadata)
}
const computePreviewText = async (categoryId, formData) => {
const category = categoryById[categoryId]
if (!category) {
throw new Error(`Unknown category ${categoryId}`)
}
return !category.computePreviewText ? 'No preview' : category.computePreviewText(formData, computePreviewText)
}
const fillComputedProps = async (category, formData) => {
if (formData == null) {
throw new Error('"formData" must be a valid object')
}
const expandedFormData = await resolveRefs(formData)
const data = await computeData(category.id, expandedFormData)
const metadata = await computeMetadata(category.id, expandedFormData)
const previewText = await computePreviewText(category.id, expandedFormData)
if (!_.isArray(metadata)) {
throw new Error('computeMetadata must return an array of strings')
}
if (!_.isString(previewText)) {
throw new Error('computePreviewText must return a string')
}
if (data == null) {
throw new Error('computeData must return a valid object')
}
return {
data,
metadata,
previewText
}
}
/**
* Creates or updates an [Element]{@link ContentManager~Element}
* @param {String} [options.itemId=] The id of the element to add
* @param {String} options.categoryId The category of the element
* @param {Object} options.formData The content of the element
* @async
* @public
* @memberOf! ContentManager
*/
const createOrUpdateCategoryItem = async ({ itemId, categoryId, formData }) => {
categoryId = categoryId && categoryId.toLowerCase()
const category = _.find(categories, { id: categoryId })
if (category == null) {
throw new Error(`Category "${categoryId}" is not a valid registered categoryId`)
}
const item = { formData, ...(await fillComputedProps(category, formData)) }
const body = transformItemApiToDb(item)
if (itemId) {
await knex('content_items')
.update(body)
.where({ id: itemId })
.then()
} else {
const newItemId = getNewItemId(category)
await knex('content_items').insert({
...body,
createdBy: 'admin',
createdOn: helpers(knex).date.now(),
id: newItemId,
categoryId
})
}
return dumpDataToFile(categoryId)
}
const categoryItemsCount = categoryId => {
let q = knex('content_items')
if (categoryId && categoryId !== 'all') {
q = q.where({ categoryId })
}
return q.count('id as count').then(([res]) => Number(res.count))
}
const deleteCategoryItems = async ids => {
if (!_.isArray(ids) || _.some(ids, id => !_.isString(id))) {
throw new Error('Expected an array of Ids to delete')
}
await knex('content_items')
.whereIn('id', ids)
.del()
.then()
return dumpAllDataToFiles()
}
const getItemDefault = async id => {
return knex('content_items')
.where({ id })
.get(0)
}
/**
* Retrieves one item
* @param {String} query Usually the id of the {@link ContentManager.Element},
* but can also be a call to a {@link ContentManager.ElementProvider}.
* @return {ContentManager.ElementProvider}
* @memberof! ContentManager
* @example
* await bp.contentManager.getItem('#!trivia-12345')
* await bp.contentManager.getItem('#trivia-random()')
*/
const getItem = async query => {
const providerRegex = /-(.+)\((.*)\)$/i
const pMatch = query.match(providerRegex)
let item
if (pMatch) {
const provider = pMatch[1].toLowerCase()
const args = pMatch[2]
const categoryName = query.substr(0, query.length - pMatch[0].length)
const fn = getItemProviders[provider]
if (!fn) {
throw new Error(`Invalid content expression "${query}", did you forget to register the "${provider}" provider?`)
}
item = await fn(knex, categoryName, args)
if (_.isArray(item)) {
throw new Error(`Provider "${provider}" returned an array instead of an object`)
}
} else {
item = await getItemDefault(query)
}
if (!item) {
return null
}
const category = _.find(categories, { id: item.categoryId })
return {
...transformItemDbToApi(item),
categoryTitle: category.title,
categorySchema: getCategorySchema(item.categoryId)
}
return item
}
const getItemsByMetadata = async metadata => {
const items = await knex('content_items')
.where('metadata', 'like', '%|' + metadata + '|%')
.then()
return transformItemDbToApi(items)
}
const loadCategory = file => {
const filePath = path.resolve(contentDir, './' + file)
// eslint-disable-next-line no-eval
const category = eval('require')(filePath) // Dynamic loading require eval for Webpack
const requiredFields = ['id', 'title', 'jsonSchema']
requiredFields.forEach(field => {
if (_.isNil(category[field])) {
throw new Error(field + ' is required but missing in Content Form file: ' + file)
}
})
category.id = category.id.toLowerCase()
if (categoryById[category.id]) {
throw new Error('There is already a form with id=' + category.id)
}
categoryById[category.id] = category
categories.push(category)
return category
}
const readDataForFile = async fileName => {
const json = await ghostManager.readFile(contentDataDir, fileName)
if (!json) {
logger.warn(`Form content file ${fileName} not found`)
return []
}
try {
const data = JSON.parse(json)
if (!Array.isArray(data)) {
throw new Error(`${fileName} expected to contain array, contents ignored`)
}
logger.info(`Read ${data.length} item(s) from ${fileName}`)
return data
} catch (err) {
logger.warn(`Error reading data from ${fileName}`, err)
return []
}
}
const loadData = async (category, fileName) => {
fileById[category.id] = fileName
logger.debug(`Loading data for ${category.id} from ${fileName}`)
let data = []
try {
data = await readDataForFile(fileName)
} catch (err) {
logger.warn(`Error reading data from ${fileName}`, err)
}
data = await Promise.map(data, async item => ({
...item,
categoryId: category.id,
id: item.id || getNewItemId(category)
}))
// TODO: use transaction
return Promise.map(data, async item =>
knex('content_items')
.insert(transformItemApiToDb(item))
.then()
)
}
const init = async () => {
if (!fs.existsSync(contentDir)) {
return
}
mkdirp.sync(contentDataDir)
await ghostManager.addRootFolder(contentDataDir, { filesGlob: '**/*.json' })
const files = await Promise.fromCallback(callback => glob('**/*.form.js', { cwd: contentDir }, callback))
// initial path, save raw props and IDs
await Promise.map(files, async file => {
try {
const category = loadCategory(file)
await loadData(category, file.replace(/\.form\.js$/, '.json'))
} catch (err) {
logger.warn('[Content Manager] Could not load Form: ' + file, err)
}
})
// second path, resolve refs
await Promise.map(categories, ({ id: categoryId }) =>
knex('content_items')
.select('id', 'formData')
.where('categoryId', categoryId)
.then()
.each(async ({ id, formData }) => {
const computedProps = await fillComputedProps(categoryById[categoryId], JSON.parse(formData))
return knex('content_items')
.where('id', id)
.update(transformItemApiToDb(computedProps))
})
)
}
/**
* @callback ElementProvider
* @memberOf! ContentManager
* @param {KnexInstance} knex An instance of Knex
* @param {String} category The name of the category
* @param {String} args A string with whatever was passed in the parans e.g. "random(25)"
* @example
const randomProvider = (knex, category, args) => {
return knex('content_items')
.where({ categoryId: category })
.orderBy(knex.raw('random()'))
.limit(1)
.get(0)
}
*/
/**
* Register a new item provider, which is used when parsing query for {@link ContentManager~getItem}
* @param {String} name The name of the provider, e.g. `random`
* @param {ContentManager.ElementProvider} fn A content provider function
* @memberOf ContentManager
* @public
* @example
// returns a random element from a given category
const randomProvider = (knex, category, args) => {
return knex('content_items')
.where({ categoryId: category })
.orderBy(knex.raw('random()'))
.limit(1)
.get(0)
}
bp.contentManager.registerGetItemProvider('random', randomProvider)
*/
const registerGetItemProvider = (name, fn) => {
name = name.toLowerCase()
getItemProviders[name] = fn
}
return {
init,
listAvailableCategories,
getCategorySchema,
createOrUpdateCategoryItem,
listCategoryItems,
categoryItemsCount,
deleteCategoryItems,
getItem,
getItemsByMetadata,
registerGetItemProvider
}
}