$v) { if (is_string($k) && str_starts_with($k, 'limit=')) { $limit = (int) substr($k, 6); } if (is_string($k) && str_starts_with($k, 'slug=')) { $slug = substr($k, 5); } if (is_string($k) && str_starts_with($k, 'batch=')) { $batch = (int) substr($k, 6); } } CLI::write('GG.deals detailed crawler started — ' . date('Y-m-d H:i:s'), 'green'); // ensure session exists $this->fsRequest(['cmd' => 'sessions.create', 'session' => self::SESSION]); CLI::write('FlareSolverr session ready: ' . self::SESSION, 'green'); // ---- pick games to crawl ---- $games = []; if ($slug !== null) { // single game $g = Mongo::collection('games')->findOne(['slug' => $slug]); if ($g !== null) { $games[] = $g; } } elseif ($batch !== null) { // expand from catalog: next N slugs not yet detailed CLI::write("Batch mode: expanding {$batch} games from catalog…", 'yellow'); $slugs = Mongo::collection('catalog') ->find(['crawled' => false], ['projection' => ['slug' => 1], 'sort' => ['_id' => 1], 'limit' => $batch]) ->toArray(); foreach ($slugs as $c) { $cslug = $c['slug']; $existing = Mongo::collection('games')->findOne(['slug' => $cslug]); if ($existing !== null) { // already detailed; mark catalog crawled and skip Mongo::collection('catalog')->updateOne(['slug' => $cslug], ['$set' => ['crawled' => true]]); continue; } $games[] = ['slug' => $cslug, 'title' => $cslug]; } } else { // refresh all games we already have $filter = []; $query = Mongo::collection('games')->find($filter, ['sort' => ['trending' => -1]]); if ($limit !== null) { $query = Mongo::collection('games')->find($filter, ['sort' => ['trending' => -1], 'limit' => $limit]); } $games = $query->toArray(); } CLI::write('Games to crawl: ' . count($games), 'yellow'); $updated = 0; $failed = 0; foreach ($games as $i => $game) { $gslug = $game['slug']; $url = 'https://gg.deals/game/' . $gslug . '/'; $result = $this->fsRequest([ 'cmd' => 'request.get', 'url' => $url, 'session' => self::SESSION, 'maxTimeout' => 30000, 'disableMedia'=> true, ]); $html = $result['solution']['response'] ?? null; if ($html === null || $html === '' || str_contains($html, 'Just a moment')) { CLI::error(" [{$i}] {$gslug}: failed (challenge or empty)"); $failed++; usleep(1000000); continue; } $parsed = $this->parseGamePage($html, $gslug, $game); if ($parsed !== null) { $this->applyToDb($gslug, $parsed); $updated++; CLI::write(" [{$i}] {$gslug}: saved (best \${$parsed['best_price']})"); } else { $failed++; } // mark catalog as processed regardless (so we don't retry endlessly) Mongo::collection('catalog')->updateOne( ['slug' => $gslug], ['$set' => ['crawled' => true, 'crawled_at' => new UTCDateTime(time() * 1000)]], ['upsert' => true] ); usleep(400000); } // destroy session when done $this->fsRequest(['cmd' => 'sessions.destroy', 'session' => self::SESSION]); CLI::write("Done: {$updated} updated, {$failed} failed.", 'green'); } /** * Parse a gg.deals game page using its JSON-LD structured data. * Returns best price + all offers from the AggregateOffer block. */ private function parseGamePage(string $html, string $slug, array $existing): ?array { // extract JSON-LD blocks if (! preg_match_all('/