ggdeals/app/Commands/CrawlGgdeals.php

<?php

declare(strict_types=1);

namespace App\Commands;

use App\Libraries\Mongo;
use CodeIgniter\CLI\BaseCommand;
use CodeIgniter\CLI\CLI;
use MongoDB\BSON\UTCDateTime;

/**
 * Crawls detailed game data from gg.deals via a persistent FlareSolverr session.
 *
 * A FlareSolverr container must be reachable (default host "flaresolverr", port 80).
 * The session is created once and reused, so the Cloudflare challenge is only
 * solved on the first request — subsequent requests are fast.
 *
 *   php spark gg:ggdeals                 → crawl details for all games we already have
 *   php spark gg:ggdeals --limit=50      → only the first 50 matched games (testing)
 *   php spark gg:ggdeals --slug=elden-ring → a single game by slug
 */
class CrawlGgdeals extends BaseCommand
{
    protected $group       = 'GGDeals';
    protected $name        = 'gg:ggdeals';
    protected $description = 'Crawls detailed prices/deals from gg.deals via FlareSolverr session.';
    protected $usage       = 'gg:ggdeals [--limit=N] [--slug=X]';

    private const FS_URL = 'http://flaresolverr:80/v1';
    private const SESSION = 'ggdeals-crawler';
    private const UA = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36';

    public function run(array $params): void
    {
        $limit = null;
        $slug = null;
        $batch = null;
        foreach ($params as $k => $v) {
            if (is_string($k) && str_starts_with($k, 'limit=')) {
                $limit = (int) substr($k, 6);
            }
            if (is_string($k) && str_starts_with($k, 'slug=')) {
                $slug = substr($k, 5);
            }
            if (is_string($k) && str_starts_with($k, 'batch=')) {
                $batch = (int) substr($k, 6);
            }
        }

        CLI::write('GG.deals detailed crawler started — ' . date('Y-m-d H:i:s'), 'green');

        // ensure session exists
        $this->fsRequest(['cmd' => 'sessions.create', 'session' => self::SESSION]);
        CLI::write('FlareSolverr session ready: ' . self::SESSION, 'green');

        // ---- pick games to crawl ----
        $games = [];

        if ($slug !== null) {
            // single game
            $g = Mongo::collection('games')->findOne(['slug' => $slug]);
            if ($g !== null) {
                $games[] = $g;
            }
        } elseif ($batch !== null) {
            // expand from catalog: next N slugs not yet detailed
            CLI::write("Batch mode: expanding {$batch} games from catalog…", 'yellow');
            $slugs = Mongo::collection('catalog')
                ->find(['crawled' => false], ['projection' => ['slug' => 1], 'sort' => ['_id' => 1], 'limit' => $batch])
                ->toArray();

            foreach ($slugs as $c) {
                $cslug = $c['slug'];
                $existing = Mongo::collection('games')->findOne(['slug' => $cslug]);
                if ($existing !== null) {
                    // already detailed; mark catalog crawled and skip
                    Mongo::collection('catalog')->updateOne(['slug' => $cslug], ['$set' => ['crawled' => true]]);
                    continue;
                }
                $games[] = ['slug' => $cslug, 'title' => $cslug];
            }
        } else {
            // refresh all games we already have
            $filter = [];
            $query = Mongo::collection('games')->find($filter, ['sort' => ['trending' => -1]]);
            if ($limit !== null) {
                $query = Mongo::collection('games')->find($filter, ['sort' => ['trending' => -1], 'limit' => $limit]);
            }
            $games = $query->toArray();
        }

        CLI::write('Games to crawl: ' . count($games), 'yellow');

        $updated = 0;
        $failed = 0;

        foreach ($games as $i => $game) {
            $gslug = $game['slug'];
            $url = 'https://gg.deals/game/' . $gslug . '/';

            $result = $this->fsRequest([
                'cmd'         => 'request.get',
                'url'         => $url,
                'session'     => self::SESSION,
                'maxTimeout'  => 30000,
                'disableMedia'=> true,
            ]);

            $html = $result['solution']['response'] ?? null;
            if ($html === null || $html === '' || str_contains($html, 'Just a moment')) {
                CLI::error("  [{$i}] {$gslug}: failed (challenge or empty)");
                $failed++;
                usleep(1000000);
                continue;
            }

            $parsed = $this->parseGamePage($html, $gslug, $game);
            if ($parsed !== null) {
                $this->applyToDb($gslug, $parsed);
                $updated++;
                CLI::write("  [{$i}] {$gslug}: saved (best \${$parsed['best_price']})");
            } else {
                $failed++;
            }

            // mark catalog as processed regardless (so we don't retry endlessly)
            Mongo::collection('catalog')->updateOne(
                ['slug' => $gslug],
                ['$set' => ['crawled' => true, 'crawled_at' => new UTCDateTime(time() * 1000)]],
                ['upsert' => true]
            );

            usleep(400000);
        }

        // destroy session when done
        $this->fsRequest(['cmd' => 'sessions.destroy', 'session' => self::SESSION]);
        CLI::write("Done: {$updated} updated, {$failed} failed.", 'green');
    }

    /**
     * Parse a gg.deals game page using its JSON-LD structured data.
     * Returns best price + all offers from the AggregateOffer block.
     */
    private function parseGamePage(string $html, string $slug, array $existing): ?array
    {
        // extract JSON-LD blocks
        if (! preg_match_all('/<script type="application\/ld\+json">(.*?)<\/script>/s', $html, $m)) {
            return null;
        }

        $product = null;
        foreach ($m[1] as $block) {
            $j = json_decode(trim($block), true);
            if (is_array($j) && ($j['@type'] ?? '') === 'Product' && isset($j['offers'])) {
                $product = $j;
                break;
            }
        }
        if ($product === null) {
            return null;
        }

        $offers = $product['offers'] ?? [];
        $bestPrice = (float) ($offers['lowPrice'] ?? 0);
        if ($bestPrice <= 0) {
            return null;
        }

        $offerList = [];
        foreach ($offers['offers'] ?? [] as $o) {
            $price = (float) ($o['price'] ?? 0);
            if ($price <= 0) {
                continue;
            }
            $seller = $o['seller']['name'] ?? 'Unknown';
            $offerList[] = [
                'price'   => $price,
                'seller'  => $seller,
                'expiry'  => isset($o['priceValidUntil'])
                    ? (strtotime($o['priceValidUntil']) ?: null)
                    : null,
            ];
        }

        return [
            'slug'       => $slug,
            'title'      => $product['name'] ?? $slug,
            'best_price' => $bestPrice,
            'high_price' => (float) ($offers['highPrice'] ?? $bestPrice),
            'offer_count'=> (int) ($offers['offerCount'] ?? count($offerList)),
            'offers'     => $offerList,
            'crawled_at' => new UTCDateTime(time() * 1000),
        ];
    }

    private function applyToDb(string $slug, array $parsed): void
    {
        $now = time();

        $existing = Mongo::collection('games')->findOne(['slug' => $slug]);
        $title = $parsed['title'] ?? ($existing['title'] ?? $slug);

        // update the game doc with the gg.deals lowest price (upsert for new games)
        $set = [
            'last_gg_crawled' => $parsed['crawled_at'],
            'gg_best_price'   => $parsed['best_price'],
            'title'           => $title,
            'slug'            => $slug,
            'source'          => 'ggdeals',
        ];
        if ($existing === null) {
            $set['genres']    = [];
            $set['platforms'] = ['pc'];
            $set['score']     = 0;
            $set['rating_pct'] = 0;
            $set['trending']  = 50;
            $set['cover']     = ['file' => '', 'from' => '#1b2838', 'to' => '#171d25'];
            $set['description'] = '';
        }
        if (! isset($existing['base_price']) || (float) $existing['base_price'] <= 0) {
            $set['base_price'] = $parsed['high_price'];
        }

        // all-time low: keep the minimum of existing and gg.deals lowest
        $currentHl = $existing['all_time_low']['price'] ?? null;
        if ($currentHl === null || $parsed['best_price'] < (float) $currentHl) {
            $set['all_time_low'] = [
                'price' => $parsed['best_price'],
                'store' => 'gg.deals',
                'kind'  => 'keyshop',
                'date'  => new UTCDateTime($now * 1000),
            ];
        }

        Mongo::collection('games')->updateOne(['slug' => $slug], ['$set' => $set], ['upsert' => true]);

        // store each offer as a deal (upsert by game+store)
        foreach ($parsed['offers'] as $o) {
            $storeCode = $this->storeCode($o['seller']);
            $expiry = $o['expiry'] !== null ? new UTCDateTime($o['expiry'] * 1000) : null;

            Mongo::collection('deals')->updateOne(
                ['game_slug' => $slug, 'store_code' => $storeCode, 'source' => 'ggdeals'],
                ['$set' => [
                    'game_slug'      => $slug,
                    'title'          => $title,
                    'store_code'     => $storeCode,
                    'kind'           => 'keyshop',
                    'price'          => round($o['price'], 2),
                    'regular'        => round($parsed['high_price'], 2),
                    'cut'            => $parsed['high_price'] > 0 ? (int) round((1 - $o['price'] / $parsed['high_price']) * 100) : 0,
                    'historical_low' => $o['price'] <= $parsed['best_price'] + 0.01,
                    'drm'            => 'GG.deals key',
                    'regional'       => false,
                    'expiry'         => $expiry,
                    'url'            => 'https://gg.deals/game/' . $slug . '/',
                    'last_crawled'   => new UTCDateTime($now * 1000),
                    'source'         => 'ggdeals',
                ]],
                ['upsert' => true]
            );
        }
    }

    /**
     * Map a gg.deals seller name to a store code (best-effort).
     */
    private function storeCode(string $seller): string
    {
        $map = [
            'steam' => 'steam', 'gog' => 'gog', 'gog.com' => 'gog',
            'epic' => 'epic', 'fanatical' => 'fanatical',
            'humble' => 'humble', 'green man gaming' => 'gmg', 'gmg' => 'gmg',
            'eneba' => 'eneba', 'kinguin' => 'kinguin', 'g2a' => 'g2a',
            'gamivo' => 'gamivo', 'cdkeys' => 'cdkeys', 'hrk game' => 'hrk',
            'instant gaming' => 'instantgaming', 'gamebillet' => 'gamebillet',
            'gamersgate' => 'gamersgate', 'gamesplanet' => 'gamesplanet',
            'k4g.com' => 'k4g', 'difmark' => 'difmark', 'driffle' => 'driffle',
        ];
        $key = strtolower(trim($seller));
        foreach ($map as $k => $code) {
            if (str_contains($key, $k)) {
                return $code;
            }
        }

        return 'gg_' . url_title($seller, '_', true);
    }

    /**
     * Send a command to the FlareSolverr API.
     */
    private function fsRequest(array $payload): ?array
    {
        $ch = curl_init(self::FS_URL);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT        => 90,
            CURLOPT_POST           => true,
            CURLOPT_POSTFIELDS     => json_encode($payload),
            CURLOPT_HTTPHEADER     => ['Content-Type: application/json'],
            CURLOPT_SSL_VERIFYPEER => false,
        ]);
        $body = curl_exec($ch);
        $code = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
        curl_close($ch);

        if ($code !== 200 || $body === false) {
            CLI::error('FlareSolverr request failed: ' . $code);

            return null;
        }

        $json = json_decode($body, true);
        if (! is_array($json)) {
            return null;
        }

        if (($json['status'] ?? '') === 'error') {
            CLI::error('FlareSolverr error: ' . ($json['message'] ?? 'unknown'));

            return null;
        }

        return $json;
    }
}