9 lines
2.3 KiB
JavaScript
9 lines
2.3 KiB
JavaScript
const fs=require('fs'),path=require('path'),root=process.env.MIGRATION_APP_ROOT||process.cwd(),dir=process.env.MIGRATION_RUN_DIR||'/var/lib/ai-chatbot/aliyun-qiniu-20261014';
|
|
require(path.join(root,'node_modules/dotenv')).config({path:path.join(root,'.env.local')});const sql=require(path.join(root,'node_modules/postgres'))(process.env.POSTGRES_URL,{max:1,prepare:false});const {S3Client,ListObjectsV2Command}=require(path.join(root,'node_modules/@aws-sdk/client-s3'));
|
|
(async()=>{let client;try{const complete=path.join(dir,'source-metadata.complete');if(fs.existsSync(complete)){console.log('Source metadata inventory already completed');return;}const plan=JSON.parse(fs.readFileSync(path.join(dir,'plan-all.json'))),wanted=new Set(plan.filter(r=>!r.variant).map(r=>r.sourceKey));
|
|
const [s]=await sql`select * from "StorageConfig" where bucket='demo-sparkai' limit 1`;client=new S3Client({endpoint:s.endpoint,region:s.region,forcePathStyle:false,credentials:{accessKeyId:s.accessKeyId,secretAccessKey:s.secretAccessKey},maxAttempts:3});let marker,listed=0,matched=0;const checkpointFile=path.join(dir,'source-list-checkpoint.json'),file=path.join(dir,'source-metadata.jsonl');if(fs.existsSync(checkpointFile)){const c=JSON.parse(fs.readFileSync(checkpointFile));marker=c.marker;listed=c.listed;matched=c.matched;}
|
|
for(;;){const r=await client.send(new ListObjectsV2Command({Bucket:s.bucket,MaxKeys:1000,ContinuationToken:marker}),{abortSignal:AbortSignal.timeout(60000)});const lines=[];for(const o of r.Contents||[]){listed++;if(wanted.has(o.Key)){lines.push(JSON.stringify({key:o.Key,size:o.Size,md5:String(o.ETag||'').replaceAll('"','').toLowerCase()}));matched++;}}
|
|
if(lines.length)fs.appendFileSync(file,lines.join('\n')+'\n',{mode:0o600});marker=r.NextContinuationToken;fs.writeFileSync(checkpointFile,JSON.stringify({marker,listed,matched}),{mode:0o600});if(listed%25000<1000)console.log(JSON.stringify({listed,matched,wanted:wanted.size}));if(!r.IsTruncated)break;}
|
|
fs.writeFileSync(complete,JSON.stringify({listed,matched,wanted:wanted.size,completedAt:new Date().toISOString()}),{mode:0o600});console.log('Source inventory complete',JSON.stringify({listed,matched,wanted:wanted.size}));
|
|
}finally{client?.destroy();await sql.end()}})().catch(e=>{console.error('source_inventory_failed',e.name,String(e.message).replace(/https?:\/\/\S+/g,'[URL]').slice(0,180));process.exitCode=1});
|