johan/FoundKey
1
0
Fork 0
forked from FoundKeyGang/FoundKey
Code Issues Pull requests Projects Releases Packages Wiki Activity

change all migrations to parallely

Browse source
This commit is contained in:
otofune 2017-11-07 23:10:38 +09:00
parent 9f2dc28088
commit 6875b68f3e
2 changed files with 45 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
tools/migration/change-gridfs-metadata-name-to-filename.js
View file

@ -19,12 +19,32 @@ async function applyNewChange (doc) {
}
async function main () {
const oldTypeDocs = await DriveFile.find({
const query = {
'metadata.name': {
$exists: true
}
}
const count = await DriveFile.count(query)
const dop = Number.parseInt(process.argv[2]) || 5
const idop = ((count - (count % dop)) / dop) + 1
return zip(
1,
async (time) => {
console.log(`${time} / ${idop}`)
const doc = await db.get('drive_files').find(query, {
limit: dop, skip: time * dop
})
return Promise.all(doc.map(applyNewChange))
},
idop
).then(a => {
const rv = []
a.forEach(e => rv.push(...e))
return rv
})
return await Promise.all(oldTypeDocs.map(applyNewChange))
}
main().then(console.dir).catch(console.error)

43
tools/migration/issue_882.js
View file

@ -1,6 +1,7 @@
// for Node.js interpret
const { default: DriveFile } = require('../../built/api/models/drive-file')
const { default: zip } = require('@prezzemolo/zip')
const migrate = async (doc) => {
const result = await DriveFile.update(doc._id, {
@ -15,30 +16,32 @@ const migrate = async (doc) => {
}
async function main() {
let i = 0;
const count = await DriveFile.count({});
const iterate = async () => {
if (i == count) return true;
console.log(`${i} / ${count}`);
const doc = (await DriveFile.find({}, { limit: 1, skip: i }))[0]
const res = await migrate(doc);
if (!res) {
return false;
} else {
i++
return await iterate();
const query = {
'metadata.type': {
$exists: true
}
}
const res = await iterate();
const count = await DriveFile.count(query);
if (res) {
return 'ok';
} else {
throw 'something happened';
}
const dop = Number.parseInt(process.argv[2]) || 5
const idop = ((count - (count % dop)) / dop) + 1
return zip(
1,
async (time) => {
console.log(`${time} / ${idop}`)
const doc = await db.get('drive_files').find(query, {
limit: dop, skip: time * dop
})
return Promise.all(doc.map(migrate))
},
idop
).then(a => {
const rv = []
a.forEach(e => rv.push(...e))
return rv
})
}
main().then(console.dir).catch(console.error)