Skip to content

Commit

Permalink
Fix databaseless configuration not intializing
Browse files Browse the repository at this point in the history
  • Loading branch information
@synzen
synzen committed Dec 31, 2018
1 parent 804f7ad commit 1167205
Show file tree
Hide file tree
Showing 4 changed files with 62 additions and 40 deletions.
1 change: 1 addition & 0 deletions structs/FeedSchedule.js
View file
Expand Up @@ -405,6 +405,7 @@ class FeedSchedule extends EventEmitter {

++this.ran

if (!config.database.uri.startsWith('mongo')) return
// Update statistics
dbOps.statistics.update({
shard: this.bot.shard && this.bot.shard.count > 0 ? this.bot.shard.id : 0,
Expand Down
7 changes: 3 additions & 4 deletions update.js
View file
@@ -1,7 +1,6 @@
// Use this to convert the deprecated old filter references to new ones on any versions past 3.0.0, including dev branch

const fs = require('fs')
const util = require('util')
const config = require('./config.js')
const storage = require('./util/storage.js')
const mongoose = require('mongoose')
Expand Down Expand Up @@ -82,7 +81,7 @@ if (config.database.uri.startsWith('mongo')) {
if (err) throw err
// Now overwrite the old with the new if necessary
const changed = updateGuildRss(guildRss)
if (!changed) console.log(`Completed ${guildRss.id} (${++c}/${fileNames.length})`)
if (!changed) console.log(`Completed ${guildRss.id} (${++c}/${fileNames.length})`)
else {
fs.writeFile(`${folderPath}/${fileName}`, JSON.stringify(guildRss, null, 2), err => {
if (err) throw err
Expand All @@ -91,7 +90,7 @@ if (config.database.uri.startsWith('mongo')) {
}
})
})
}
}
})
}

Expand Down Expand Up @@ -230,4 +229,4 @@ function updateGuildRss (guildRss) {
}

return changed
}
}
92 changes: 57 additions & 35 deletions util/dbOps.js
View file
Expand Up @@ -16,25 +16,26 @@ const readFilePromise = util.promisify(fs.readFile)
const writeFilePromise = util.promisify(fs.writeFile)
const mkdirPromise = util.promisify(fs.mkdir)
const unlinkPromise = util.promisify(fs.unlink)
const MONGO_DATABASE = config.database.uri.startsWith('mongo')

exports.guildRss = {
get: async id => {
if (config.database.uri.startsWith('mongo')) return models.GuildRss().findOne({ id }, FIND_PROJECTION).lean().exec()
if (MONGO_DATABASE) return models.GuildRss().findOne({ id }, FIND_PROJECTION).lean().exec()
return JSON.parse(await readFilePromise(path.join(config.database.uri, `${id}.json`)))
},
getMany: async ids => {
if (config.database.uri.startsWith('mongo')) return models.GuildRss().find({ id: { $in: ids } }, FIND_PROJECTION).lean().exec()
if (MONGO_DATABASE) return models.GuildRss().find({ id: { $in: ids } }, FIND_PROJECTION).lean().exec()
const promises = []
for (const id of ids) promises.push(exports.guildRss.get(id))
return Promise.all(promises)
},
getAll: async () => {
// Database version
if (config.database.uri.startsWith('mongo')) return models.GuildRss().find({}, FIND_PROJECTION).lean().exec()
if (MONGO_DATABASE) return models.GuildRss().find({}, FIND_PROJECTION).lean().exec()

// Memory Version
if (!fs.existsSync(path.join(config.database.uri))) return []
const fileNames = await readdirPromise(path.join(config.database.uri))
const fileNames = (await readdirPromise(path.join(config.database.uri))).filter(fileName => /^\d+$/.test(fileName.replace(/\.json/i, '')))
return new Promise((resolve, reject) => { // Avoid await for the below to allow async reads
let done = 0
let read = []
Expand All @@ -51,17 +52,19 @@ exports.guildRss = {
log.general.warning(`Could not parse JSON from source file ${name}`, err)
}
if (++done === total) resolve(read)
console.log(done, total)
})
.catch(err => {
log.general.warning(`Could not read source file ${name}`, err)
if (++done === total) resolve(read)
console.log(done, total)
})
}
})
},
update: async (guildRss, skipEmptyCheck) => {
// Memory version
if (!config.database.uri.startsWith('mongo')) {
if (!MONGO_DATABASE) {
try {
if (!fs.existsSync(path.join(config.database.uri))) await mkdirPromise(path.join(config.database.uri))
await writeFilePromise(path.join(config.database.uri, `${guildRss.id}.json`), JSON.stringify(guildRss, null, 2))
Expand All @@ -81,7 +84,7 @@ exports.guildRss = {
const guildId = guildRss.id
if (guildRss && guildRss.sources && Object.keys(guildRss.sources).length > 0) exports.guildRss.backup(guildRss).catch(err => log.general.warning('Unable to backup guild after remvoing', err, true))
// Memory version
if (!config.database.uri.startsWith('mongo')) {
if (!MONGO_DATABASE) {
await unlinkPromise(path.join(config.database.uri, `${guildId}.json`))
const rssList = guildRss ? guildRss.sources : undefined
if (rssList) {
Expand Down Expand Up @@ -126,7 +129,7 @@ exports.guildRss = {
backup: async guildRss => {
if (!guildRss || exports.guildRss.empty(guildRss, true)) return
// Memory version
if (!config.database.uri.startsWith('mongo')) {
if (!MONGO_DATABASE) {
if (!fs.existsSync(path.join(config.database.uri, 'backup'))) {
await mkdirPromise(path.join(config.database.uri, 'backup'))
}
Expand All @@ -138,7 +141,7 @@ exports.guildRss = {
restore: async guildId => {
// Memory version
let guildRss
if (!config.database.uri.startsWith('mongo')) {
if (!MONGO_DATABASE) {
if (!fs.existsSync(path.join(config.database.uri, 'backup', `${guildId}.json`))) return
try {
const json = await readFilePromise(path.join(config.database.uri, 'backup', `${guildId}.json`))
Expand Down Expand Up @@ -194,7 +197,7 @@ exports.feeds = {
exports.linkTracker = {
write: async linkTracker => {
if (!(linkTracker instanceof LinkTracker)) throw new TypeError('linkTracker argument is not instance of LinkTracker')
if (!config.database.uri.startsWith('mongo')) return
if (!MONGO_DATABASE) return
try {
await models.LinkTracker().collection.drop().catch(err => {
if (err.code !== 26) log.general.warning('Unable to drop link tracker collection before insertion', err, true)
Expand All @@ -206,12 +209,12 @@ exports.linkTracker = {
}
},
get: async () => {
if (!config.database.uri.startsWith('mongo')) return new LinkTracker()
if (!MONGO_DATABASE) return new LinkTracker()
const docs = await models.LinkTracker().find({}).lean().exec()
return new LinkTracker(docs, storage.bot)
},
update: async (link, count, scheduleName) => {
if (!config.database.uri.startsWith('mongo')) return
if (!MONGO_DATABASE) return
const shardId = storage.bot && storage.bot.shard && storage.bot.shard.count > 0 ? storage.bot.shard.id : undefined
if (count > 0) {
await models.LinkTracker().updateOne({ link: link, shard: shardId, scheduleName: scheduleName }, { $set: { link: link, count: count, shard: shardId, scheduleName: scheduleName } }, UPDATE_SETTINGS).exec()
Expand All @@ -224,7 +227,7 @@ exports.linkTracker = {
}
},
decrement: async (link, scheduleName) => {
if (!config.database.uri.startsWith('mongo')) return
if (!MONGO_DATABASE) return
const linkTracker = await exports.linkTracker.get()
if (!linkTracker.get(link, scheduleName)) return
const count = linkTracker.decrement(link, scheduleName)
Expand All @@ -239,7 +242,7 @@ exports.linkTracker = {
await exports.linkTracker.update(link, count, scheduleName)
},
increment: async (link, scheduleName) => {
if (!config.database.uri.startsWith('mongo')) return
if (!MONGO_DATABASE) return
const shard = storage.bot && storage.bot.shard && storage.bot.shard.count > 0 ? storage.bot.shard.id : undefined
await models.LinkTracker().updateOne({ link, shard, scheduleName }, { $inc: { count: 1 } }, UPDATE_SETTINGS).exec()
}
Expand Down Expand Up @@ -281,56 +284,65 @@ exports.failedLinks = {
})
.catch(err => log.general.warning(`Failed to query all profiles for _sendAlert for failed link ${link}`, err))
},
get: async link => models.FailedLink().findOne({ link }, FIND_PROJECTION).lean().exec(),
getMultiple: async links => models.FailedLink().find({ link: { $in: links } }, FIND_PROJECTION).lean().exec(),
getAll: async () => models.FailedLink().find({}, FIND_PROJECTION).lean().exec(),
get: async link => {
if (!MONGO_DATABASE) return
return models.FailedLink().findOne({ link }, FIND_PROJECTION).lean().exec()
},
getMultiple: async links => {
if (!MONGO_DATABASE) return []
return models.FailedLink().find({ link: { $in: links } }, FIND_PROJECTION).lean().exec()
},
getAll: async () => {
if (!MONGO_DATABASE) return []
return models.FailedLink().find({}, FIND_PROJECTION).lean().exec()
},
increment: async link => {
if (FAIL_LIMIT === 0 || !config.database.uri.startsWith('mongo')) return
if (FAIL_LIMIT === 0 || !MONGO_DATABASE) return
await storage.models.FailedLink().updateOne({ link: link }, { $inc: { count: 1 } }, UPDATE_SETTINGS).exec()
},
fail: async link => {
if (!config.database.uri.startsWith('mongo')) return
if (!MONGO_DATABASE) return
if (config.feeds.failLimit === 0) throw new Error('Unable to fail a link when config.feeds.failLimit is 0')
const now = new Date().toString()
if (config.feeds.notifyFail === true) exports.failedLinks._sendAlert(link, `**ATTENTION** - Feed link <${link}> has reached the connection failure limit and will not be retried until it is manually refreshed by this server, or another server using this feed. See \`${config.bot.prefix}rsslist\` for more information.`)
await storage.models.FailedLink().updateOne({ link: link }, { $set: { link: link, failed: now } }, UPDATE_SETTINGS).exec()
log.cycle.error(`${link} has been failed and will no longer be retrieved on subsequent retrieval cycles`)
},
reset: async link => {
if (!config.database.uri.startsWith('mongo')) return
if (!MONGO_DATABASE) return
await storage.models.FailedLink().deleteOne({ link: link })
}
}

exports.blacklists = {
uniformize: async (blacklistGuilds, blacklistUsers, skipProcessSend) => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.blacklists.uniformize not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.blacklists.uniformize not supported when config.database.uri is set to a databaseless folder path')
if (!skipProcessSend && storage.bot.shard && storage.bot.shard.count > 0) process.send({ _drss: true, type: 'blacklists.uniformize', blacklistGuilds: blacklistGuilds, blacklistUsers: blacklistUsers, _loopback: true })
else if (skipProcessSend) {
storage.blacklistGuilds = blacklistGuilds
storage.blacklistUsers = blacklistUsers
}
},
getAll: async () => {
if (!config.database.uri.startsWith('mongo')) return []
if (!MONGO_DATABASE) return []
return models.Blacklist().find().lean().exec()
},
add: async settings => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.blacklists.add is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.blacklists.add is not supported when config.database.uri is set to a databaseless folder path')
await models.Blacklist().updateOne({ id: settings.id }, { $set: settings }, UPDATE_SETTINGS).exec()
if (settings.isGuild) storage.blacklistGuilds.push(settings.id)
else storage.blacklistUsers.push(settings.id)
await exports.blacklists.uniformize(storage.blacklistGuilds, storage.blacklistUsers)
},
remove: async id => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.blacklists.remove is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.blacklists.remove is not supported when config.database.uri is set to a databaseless folder path')
const doc = await models.Blacklist().deleteOne({ id: id })
if (storage.blacklistGuilds.includes(id)) storage.blacklistGuilds.splice(storage.blacklistGuilds.indexOf(doc.id), 1)
else storage.blacklistUsers.splice(storage.blacklistUsers.indexOf(doc.id), 1)
await exports.blacklists.uniformize(storage.blacklistGuilds, storage.blacklistUsers)
},
refresh: async () => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.blacklists.refresh is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.blacklists.refresh is not supported when config.database.uri is set to a databaseless folder path')
const docs = await exports.blacklists.getAll()
for (var x = 0; x < docs.length; ++x) {
const doc = docs[x]
Expand All @@ -342,10 +354,20 @@ exports.blacklists = {
}

exports.statistics = {
clear: () => models.Statistics().collection.drop(),
get: (shard = 0) => models.Statistics().findOne({ shard }, FIND_PROJECTION).lean().exec(),
getAll: () => models.Statistics().find({}, FIND_PROJECTION).lean().exec(),
clear: async () => {
if (!MONGO_DATABASE) return
return models.Statistics().collection.drop()
},
get: async (shard = 0) => {
if (!MONGO_DATABASE) return
return models.Statistics().findOne({ shard }, FIND_PROJECTION).lean().exec()
},
getAll: async () => {
if (!MONGO_DATABASE) return []
return models.Statistics().find({}, FIND_PROJECTION).lean().exec()
},
update: async data => {
if (!MONGO_DATABASE) return
const shard = data.shard || 0
const shardStats = await exports.statistics.get(shard)
if (!shardStats) return models.Statistics().updateOne({ shard }, { $set: data }, UPDATE_SETTINGS).exec()
Expand All @@ -360,11 +382,11 @@ exports.statistics = {
exports.vips = {
get: id => models.VIP().findOne({ id }, FIND_PROJECTION).lean().exec(),
getAll: async () => {
if (!config.database.uri.startsWith('mongo')) return []
if (!MONGO_DATABASE) return []
return models.VIP().find({}, FIND_PROJECTION).lean().exec()
},
update: async settings => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.vips.update is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.vips.update is not supported when config.database.uri is set to a databaseless folder path')
if (!settings.name) {
const dUser = storage.bot.users.get(settings.id)
settings.name = dUser ? dUser.username : null
Expand All @@ -373,7 +395,7 @@ exports.vips = {
log.general.success(`Updated VIP ${settings.id} (${settings.name})`)
},
updateBulk: async vipUsers => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.vips.updateBulk is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.vips.updateBulk is not supported when config.database.uri is set to a databaseless folder path')
let complete = 0
const total = Object.keys(vipUsers).length
let errored = false
Expand Down Expand Up @@ -403,11 +425,11 @@ exports.vips = {
})
},
remove: async id => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.vips.remove is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.vips.remove is not supported when config.database.uri is set to a databaseless folder path')
await models.VIP().deleteOne({ id: id }).exec()
},
addServers: async settings => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.vips.addServers is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.vips.addServers is not supported when config.database.uri is set to a databaseless folder path')
const { serversToAdd, vipUser } = settings
if (serversToAdd.length === 0) return
for (const id of serversToAdd) {
Expand All @@ -418,7 +440,7 @@ exports.vips = {
await models.VIP().updateOne({ id: vipUser.id }, { $addToSet: { servers: { $each: serversToAdd } } }, UPDATE_SETTINGS).exec()
},
removeServers: async settings => {
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.vips.removeServers is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.vips.removeServers is not supported when config.database.uri is set to a databaseless folder path')
const { serversToRemove, vipUser } = settings
for (const id of serversToRemove) {
const index = vipUser.servers.indexOf(id)
Expand All @@ -443,15 +465,15 @@ exports.vips = {
},
refresh: async () => {
if (!config._vip) return
if (!config.database.uri.startsWith('mongo')) throw new Error('dbOps.vips.refresh is not supported when config.database.uri is set to a databaseless folder path')
if (!MONGO_DATABASE) throw new Error('dbOps.vips.refresh is not supported when config.database.uri is set to a databaseless folder path')
if (!fs.existsSync(path.join(__dirname, '..', 'settings', 'vips.js'))) throw new Error('Missing VIP module')
return require('../settings/vips.js')(storage.bot)
}
}

exports.general = {
cleanDatabase: async currentCollections => { // Remove unused feed collections
if (!config.database.uri.startsWith('mongo')) return
if (!MONGO_DATABASE) return
if (!Array.isArray(currentCollections)) throw new Error('currentCollections is not an Array')
const names = await mongoose.connection.db.listCollections().toArray()
let c = 0
Expand Down
2 changes: 1 addition & 1 deletion util/initialization.js
View file
Expand Up @@ -171,7 +171,7 @@ module.exports = async (bot, customSchedules, callback) => {

log.init.info(`${SHARD_ID}Finished initialization`)
if (!bot.shard || bot.shard.count === 0) {
dbOps.statistics.clear().catch(err => err.code === 26 ? null : log.general.warning('Unable to drop statistics database', err)) // 26 is ns not found - it's fine if it didn't exist in the first place
if (config.database.uri.startsWith('mongo')) dbOps.statistics.clear().catch(err => err.code === 26 ? null : log.general.warning('Unable to drop statistics database', err)) // 26 is ns not found - it's fine if it didn't exist in the first place
dbOps.linkTracker.write(linkTracker).catch(err => log.general.warning('Unable to write link tracker links to collection after initialization', err)) // If this is a shard, then it's handled by the sharding manager
dbOps.general.cleanDatabase(currentCollections)
.then(() => callback(missingGuilds, linkTracker.toDocs(), feedData))
Expand Down

0 comments on commit 1167205

Please sign in to comment.