我想把大量的对象写到谷歌云存储桶中。我以50个为一批的方式并行执行这些写入,每批写入之间有1秒的延迟。
这是我的密码。
const { Storage } = require("@google-cloud/storage");
const keyFilename = "path/to/service/account/file";
const projectId = "projectId";
const googleCloudConfig = { projectId, keyFilename };
const storage = new Storage(googleCloudConfig);
const bucket = storage.bucket("bucketName");
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
const writeDocs = async () => {
try {
const arr = new Array(1000).fill({ test: "test"});
const promises = [];
for (let i=0; i < arr.length; i++) {
const file = bucket.file(`test/${i}.json`);
promises.push(file.save(JSON.stringify(arr[i]), () => console.log(`saved JSON document ${i} to storage`)));
if (promises.length >= 50) {
console.log("writing batch. total:", i+1)
await Promise.all(promises);
promises.length = 0;
await sleep(1000);
}
}
if (promises.length) {
await Promise.all(promises);
}
} catch (error) {
console.error(error);
}
}
writeDocs();
我希望在中有1000个对象
test/
目录,但只有400。为什么会这样?这里有没有相关的未记录的利率限制?
编辑:我已经编辑了我的问题,只包括一个演示这个问题的最小例子。