joinlemmy-site/crawl.mjs

import fs from 'fs';
import path from 'path';
import { exit } from 'process';
import { spawn } from 'child_process';

const outDir = "src/shared/translations/";
const recommendationsFile = "recommended-instances.json";
const instanceStatsFile = "src/shared/instance_stats.ts";

fs.mkdirSync(outDir, { recursive: true });

// crawl instance stats
try {
  const recommended_instances = JSON.parse(fs.readFileSync(recommendationsFile, "utf8"));
  var all_recommended = [];
  for (var k in recommended_instances) {
    if (k != "exclude") {
      all_recommended.push(...recommended_instances[k]);
    }
  }
  const run = spawn("cargo",
      ["run", "--", "--start-instances", all_recommended,
          "--exclude-instances", recommended_instances.exclude], {
        cwd: "lemmy-stats-crawler",
        encoding : 'utf8'
      });
  let savedOutput = '';

  run.stdout.on('data', data => {
     const strData = data.toString();
     process.stdout.write(strData);
     savedOutput += strData;
  });

  run.stderr.on('data', data => {
     const strData = data.toString();
     process.stdout.write(strData);
  });

  run.on('close', exitCode => {
    const stats = JSON.parse(savedOutput);

    let stats2 = {
      stats: stats,
      recommended: recommended_instances
    }

    let data = `export const instance_stats = \n `;
    data += JSON.stringify(stats2, null, 2) + ";";
    fs.writeFileSync(instanceStatsFile, data);
  });
    run.await;
} catch (err) {
  console.error(err);
}