I have written this code out in many different ways. I FINALLY got it to work within the limit rates, but now my search is not returning matches. I am just trying to find ANY image filename in the entire account that includes a sku from the database.
The code runs fine, just does not find any matches that I know exists in the account.
Any advice helps.
const AWS = require('aws-sdk');
const fs = require('fs');
const mysql = require('mysql2/promise');
const path = require('path');
const Dropbox = require('dropbox').Dropbox;
const throttledQueue = require('throttled-queue');
const dbx = new Dropbox({
accessToken:
'#',
});
const pool = mysql.createPool({
connectionLimit: 16,
host: 'localhost',
user: 'root',
password: '#',
database: 'absdropbox',
});
const throttledRequestsPerSecond = 15;
const throttle = throttledQueue(throttledRequestsPerSecond, 1000);
async function getSKUs() {
const [rows] = await pool.query('SELECT sku FROM oe');
return rows.map((row) => row.sku);
}
async function queryDropbox(query) {
const options = {
path: '',
// path: '/home/2022',
mode: 'filename_and_content',
query: query,
};
try {
const response = await dbx.filesSearchV2(options);
return response.matches ?? [];
} catch (error) {
console.error(error);
return [];
}
}
let i = 0
async function processSKU(sku) {
// const query = `filename LIKE '%${sku}%'`;
// const query = `filename LIKE '_${sku}_'`;
const query = `search('${sku}')`;
const matches = await queryDropbox(query);
console.log(i++)
if (matches.length === 0) {
console.log(`No matches found for SKU ${sku}`);
return;
}
console.log(`Found ${matches.length} match(es) for SKU ${sku}`);
for (const match of matches) {
const { path_display } = match.metadata;
const filename = path.basename(path_display);
const fileStream = fs.createWriteStream(filename);
try {
const { data } = await dbx.filesDownload({ path: path_display });
data.pipe(fileStream);
await new Promise((resolve, reject) => {
fileStream.on('finish', resolve);
fileStream.on('error', reject);
});
const s3 = new AWS.S3({ region: 'us-east-2' });
const s3Params = {
Bucket: 'dbexctract',
Key: filename,
Body: fs.createReadStream(filename),
};
await s3.upload(s3Params).promise();
console.log(`File ${filename} uploaded to S3`);
} catch (error) {
console.error(error);
} finally {
fs.unlinkSync(filename);
}
}
}
async function run() {
const skus = await getSKUs();
for (const sku of skus) {
await throttle(() => processSKU(sku));
}
}
run();
