GG.deals clone: CodeIgniter 4 + MongoDB game price tracker

- Full gg.deals-style UI: deal feed with filters, game pages with price history charts, rankings, vouchers, news, prepaids, subscriptions, bundles, franchises, wishlist, API docs - Real data crawlers: * gg:crawl (Steam Store API + GOG Catalog + Epic free games) * gg:catalog (252K game slugs from gg.deals sitemaps) * gg:ggdeals (per-game prices via FlareSolverr session, JSON-LD parsing) * gg:seed / gg:crawl-covers (demo seed + Steam CDN cover art) - 24/7 self-updating scheduler (scripts/scheduler_loop.php, lockfile-protected) - Tailwind v4 dark/light theme, Alpine.js, Chart.js
2026-08-23 07:06:02 +00:00
commit fff7a4b9b9
146 changed files with 15691 additions and 0 deletions
@@ -0,0 +1,126 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Commands;
+
+use App\Libraries\Mongo;
+use CodeIgniter\CLI\BaseCommand;
+use CodeIgniter\CLI\CLI;
+use MongoDB\BSON\UTCDateTime;
+
+/**
+ * Crawls the full gg.deals game catalog from the public sitemap files.
+ *
+ * The sitemap XML files are NOT Cloudflare-protected, so they can be fetched
+ * directly. There are 526+ sitemap_games_N.xml files, each with ~990 game URLs.
+ *
+ *   php spark gg:catalog                → crawl all sitemap files into the catalog collection
+ *   php spark gg:catalog --limit=5      → only first N files (for testing)
+ */
+class CrawlCatalog extends BaseCommand
+{
+    protected $group       = 'GGDeals';
+    protected $name        = 'gg:catalog';
+    protected $description = 'Crawls the full gg.deals game catalog from public sitemaps.';
+    protected $usage       = 'gg:catalog [--limit=N]';
+
+    private const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36';
+
+    public function run(array $params): void
+    {
+        CLI::write('GG.deals catalog crawler started — ' . date('Y-m-d H:i:s'), 'green');
+
+        $limit = null;
+        foreach ($params as $k => $v) {
+            if (is_string($k) && str_starts_with($k, 'limit=')) {
+                $limit = (int) substr($k, 6);
+            }
+        }
+
+        // resolve the sitemap index
+        $index = $this->fetch('https://gg.deals/sitemap.xml');
+        if ($index === null) {
+            CLI::error('Could not fetch sitemap index');
+
+            return;
+        }
+
+        preg_match_all('/sitemap_games_(\d+)\.xml/', $index, $m);
+        $files = array_map(static fn ($n) => (int) $n, $m[1]);
+        sort($files, SORT_NUMERIC);
+
+        CLI::write('Found ' . count($files) . ' game sitemap files.', 'yellow');
+        if ($limit !== null) {
+            $files = array_slice($files, 0, $limit);
+            CLI::write('Limit set: first ' . count($files) . ' files.', 'yellow');
+        }
+
+        $totalUrls = 0;
+        $totalInserted = 0;
+        $totalSkipped = 0;
+
+        foreach ($files as $i => $n) {
+            $url = "https://gg.deals/sitemap_games_{$n}.xml";
+            $xml = $this->fetch($url);
+            if ($xml === null) {
+                CLI::error("  sitemap {$n}: fetch failed");
+                usleep(500000);
+                continue;
+            }
+
+            preg_match_all('/<loc>(https:\/\/gg\.deals\/game\/([a-z0-9-]+)\/?)<\/loc>/', $xml, $matches);
+            $slugs = array_values(array_unique($matches[2]));
+            $totalUrls += count($slugs);
+
+            $docs = [];
+            foreach ($slugs as $slug) {
+                $docs[] = [
+                    'slug'         => $slug,
+                    'url'          => 'https://gg.deals/game/' . $slug . '/',
+                    'crawled'      => false,
+                    'added_at'     => new UTCDateTime(time() * 1000),
+                ];
+            }
+
+            if ($docs !== []) {
+                try {
+                    Mongo::collection('catalog')->insertMany($docs, ['ordered' => false]);
+                    $totalInserted += count($docs);
+                } catch (\MongoDB\Driver\Exception\BulkWriteException $e) {
+                    // duplicate keys are expected on re-runs
+                    $totalInserted += count($docs);
+                }
+            }
+
+            CLI::write("  [{$i}/" . count($files) . "] sitemap {$n}: " . count($slugs) . ' slugs (total ' . number_format($totalInserted) . ')');
+            usleep(300000);
+        }
+
+        CLI::write('');
+        CLI::write("Catalog crawl done: {$totalUrls} urls, {$totalInserted} inserted.", 'green');
+        CLI::write('Total catalog documents: ' . Mongo::collection('catalog')->countDocuments(), 'green');
+    }
+
+    private function fetch(string $url): ?string
+    {
+        $ch = curl_init($url);
+        curl_setopt_array($ch, [
+            CURLOPT_RETURNTRANSFER => true,
+            CURLOPT_TIMEOUT        => 30,
+            CURLOPT_FOLLOWLOCATION => true,
+            CURLOPT_USERAGENT      => self::UA,
+            CURLOPT_SSL_VERIFYPEER => true,
+            CURLOPT_HTTPHEADER     => ['Accept: application/xml,text/xml,*/*'],
+        ]);
+        $body = curl_exec($ch);
+        $code = (int) curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
+        curl_close($ch);
+
+        if ($code !== 200 || $body === false || $body === '') {
+            return null;
+        }
+
+        return $body;
+    }
+}