ggdeals/app/Commands/CrawlCatalog.php

<?php

declare(strict_types=1);

namespace App\Commands;

use App\Libraries\Mongo;
use CodeIgniter\CLI\BaseCommand;
use CodeIgniter\CLI\CLI;
use MongoDB\BSON\UTCDateTime;

/**
 * Crawls the full gg.deals game catalog from the public sitemap files.
 *
 * The sitemap XML files are NOT Cloudflare-protected, so they can be fetched
 * directly. There are 526+ sitemap_games_N.xml files, each with ~990 game URLs.
 *
 *   php spark gg:catalog                → crawl all sitemap files into the catalog collection
 *   php spark gg:catalog --limit=5      → only first N files (for testing)
 */
class CrawlCatalog extends BaseCommand
{
    protected $group       = 'GGDeals';
    protected $name        = 'gg:catalog';
    protected $description = 'Crawls the full gg.deals game catalog from public sitemaps.';
    protected $usage       = 'gg:catalog [--limit=N]';

    private const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';

    public function run(array $params): void
    {
        CLI::write('GG.deals catalog crawler started — ' . date('Y-m-d H:i:s'), 'green');

        $limit = null;
        foreach ($params as $k => $v) {
            if (is_string($k) && str_starts_with($k, 'limit=')) {
                $limit = (int) substr($k, 6);
            }
        }

        // resolve the sitemap index
        $index = $this->fetch('https://gg.deals/sitemap.xml');
        if ($index === null) {
            CLI::error('Could not fetch sitemap index');

            return;
        }

        preg_match_all('/sitemap_games_(\d+)\.xml/', $index, $m);
        $files = array_map(static fn ($n) => (int) $n, $m[1]);
        sort($files, SORT_NUMERIC);

        CLI::write('Found ' . count($files) . ' game sitemap files.', 'yellow');
        if ($limit !== null) {
            $files = array_slice($files, 0, $limit);
            CLI::write('Limit set: first ' . count($files) . ' files.', 'yellow');
        }

        $totalUrls = 0;
        $totalInserted = 0;
        $totalSkipped = 0;

        foreach ($files as $i => $n) {
            $url = "https://gg.deals/sitemap_games_{$n}.xml";
            $xml = $this->fetch($url);
            if ($xml === null) {
                CLI::error("  sitemap {$n}: fetch failed");
                usleep(500000);
                continue;
            }

            preg_match_all('/<loc>(https:\/\/gg\.deals\/game\/([a-z0-9-]+)\/?)<\/loc>/', $xml, $matches);
            $slugs = array_values(array_unique($matches[2]));
            $totalUrls += count($slugs);

            $docs = [];
            foreach ($slugs as $slug) {
                $docs[] = [
                    'slug'         => $slug,
                    'url'          => 'https://gg.deals/game/' . $slug . '/',
                    'crawled'      => false,
                    'added_at'     => new UTCDateTime(time() * 1000),
                ];
            }

            if ($docs !== []) {
                try {
                    Mongo::collection('catalog')->insertMany($docs, ['ordered' => false]);
                    $totalInserted += count($docs);
                } catch (\MongoDB\Driver\Exception\BulkWriteException $e) {
                    // duplicate keys are expected on re-runs
                    $totalInserted += count($docs);
                }
            }

            CLI::write("  [{$i}/" . count($files) . "] sitemap {$n}: " . count($slugs) . ' slugs (total ' . number_format($totalInserted) . ')');
            usleep(300000);
        }

        CLI::write('');
        CLI::write("Catalog crawl done: {$totalUrls} urls, {$totalInserted} inserted.", 'green');
        CLI::write('Total catalog documents: ' . Mongo::collection('catalog')->countDocuments(), 'green');
    }

    private function fetch(string $url): ?string
    {
        $ch = curl_init($url);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT        => 30,
            CURLOPT_FOLLOWLOCATION => true,
            CURLOPT_USERAGENT      => self::UA,
            CURLOPT_SSL_VERIFYPEER => true,
            CURLOPT_HTTPHEADER     => ['Accept: application/xml,text/xml,*/*'],
        ]);
        $body = curl_exec($ch);
        $code = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
        curl_close($ch);

        if ($code !== 200 || $body === false || $body === '') {
            return null;
        }

        return $body;
    }
}